From 59959b51180c06840960157dca60077963293278 Mon Sep 17 00:00:00 2001 From: temporal Date: Thu, 10 Jul 2008 02:12:20 +0000 Subject: [PATCH 001/525] Initial checkin. From dd24b8b9ae5f822409ff30c75a368234ebaedc29 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 18 Jan 2010 22:18:18 -0800 Subject: [PATCH 002/525] initial commit --- .gitignore | 2 + README | 0 src/clj/protobuf.clj | 10 + .../protobuf/PersistentProtocolBufferMap.java | 267 ++++++++++++++++++ 4 files changed, 279 insertions(+) create mode 100644 .gitignore create mode 100644 README create mode 100644 src/clj/protobuf.clj create mode 100644 src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cb290e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.class +*.jar \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..e69de29 diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj new file mode 100644 index 0000000..f6a0042 --- /dev/null +++ b/src/clj/protobuf.clj @@ -0,0 +1,10 @@ +(ns protobuf) + +(defmacro defprotobuf [sym & args] + (let [class (symbol (apply str (interpose "$" (map name args))))] + `(def ~sym (. ~class getDescriptor)))) + +(defn protobuf [desc & args] + (if (= (count args) 1) + (clojure.protobuf.PersistentProtocolBufferMap/create desc (first args)) + (clojure.protobuf.PersistentProtocolBufferMap/construct desc (apply hash-map args)))) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java new file mode 100644 index 0000000..b91ffdd --- /dev/null +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -0,0 +1,267 @@ +/** + * Copyright (c) Justin Balthrop. All rights reserved. + * The use and distribution terms for this software are covered by the + * Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) + * which can be found in the file epl-v10.html at the root of this distribution. + * By using this software in any fashion, you are agreeing to be bound by + * the terms of this license. + * You must not remove this notice, or any other, from this software. + **/ + +/* Based on rhickey's clojure.lang.PersistentStructMap - Jan 17, 2010 */ + +package clojure.protobuf; + +import clojure.lang.*; +import java.util.*; +import java.io.InputStream; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Descriptors; + +public class PersistentProtocolBufferMap extends APersistentMap { + final Descriptors.Descriptor type; + final DynamicMessage message; + final DynamicMessage.Builder builder; + + static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, byte[] bytes) + throws com.google.protobuf.InvalidProtocolBufferException + { + DynamicMessage message = DynamicMessage.parseFrom(type, bytes); + return new PersistentProtocolBufferMap(null, type, message, null); + } + + static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, String string) + throws com.google.protobuf.InvalidProtocolBufferException + { + return create(type, string.getBytes()); + } + + static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, InputStream input) + throws com.google.protobuf.InvalidProtocolBufferException, java.io.IOException + { + DynamicMessage message = DynamicMessage.parseFrom(type, input); + return new PersistentProtocolBufferMap(null, type, message, null); + } + + static public PersistentProtocolBufferMap construct(Descriptors.Descriptor type, IPersistentMap keyvals){ + DynamicMessage.Builder builder = DynamicMessage.newBuilder(type); + PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, type, null, builder); + return (PersistentProtocolBufferMap) protobuf.cons(keyvals); + } + + protected PersistentProtocolBufferMap(IPersistentMap meta, Descriptors.Descriptor type, + DynamicMessage message, DynamicMessage.Builder builder) { + super(meta); + this.type = type; + this.message = message; + this.builder = builder; + } + + protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Descriptors.Descriptor type, + DynamicMessage message, DynamicMessage.Builder builder) { + return new PersistentProtocolBufferMap(meta, type, message, builder); + } + + protected DynamicMessage message() { + if (message == null) { + return builder.build(); + } else { + return message; + } + } + + protected DynamicMessage.Builder builder() { + if (builder == null) { + return message.toBuilder(); + } else { + return builder.clone(); + } + } + + protected Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { + return type.findFieldByName(key.getName()); + } + + protected Object fromFieldValue(Descriptors.FieldDescriptor.JavaType type, Object value) { + switch (type) { + case ENUM: + break; + case MESSAGE: + break; + default: + return value; + } + return null; + } + + protected Object toFieldValue(Descriptors.FieldDescriptor.JavaType type, Object value) { + switch (type) { + case LONG: + if (value instanceof Long) return value; + Integer i = (Integer) value; + return new Long(i.longValue()); + case INT: + return (Integer)value; + case ENUM: + break; + case MESSAGE: + break; + default: + return value; + } + return null; + } + + protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { + if (field == null) return; + + if (field.isRepeated()) { + builder.clearField(field); + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Object value = toFieldValue(field.getJavaType(), s.first()); + builder.addRepeatedField(field, value); + } + } else { + Object value = toFieldValue(field.getJavaType(), val); + builder.setField(field, value); + } + } + + public Obj withMeta(IPersistentMap meta) { + if (meta == meta()) return this; + return makeNew(meta, type, message, builder); + } + + public boolean containsKey(Object key) { + Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + return message().hasField(field); + } + + public IMapEntry entryAt(Object key) { + Object value = valAt(key); + return (value == null) ? null : new MapEntry(key, value); + } + + public Object valAt(Object key) { + Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + if (field == null) return null; + + DynamicMessage message = message(); + + if (field.isRepeated()) { + int num_items = message.getRepeatedFieldCount(field); + List items = new ArrayList(num_items); + + for (int i = 0; i < num_items; i++) { + Object value = fromFieldValue(field.getJavaType(), message.getRepeatedField(field, i)); + items.add(value); + } + return PersistentVector.create(items); + } else { + return fromFieldValue(field.getJavaType(), message.getField(field)); + } + } + + public Object valAt(Object key, Object notFound) { + Object val = valAt(key); + return (val == null) ? notFound : val; + } + + public IPersistentMap assoc(Object key, Object val) { + DynamicMessage.Builder builder = builder(); + Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + + if (field == null) return this; + setField(builder, field, val); + return makeNew(meta(), type, null, builder); + } + + public IPersistentMap assocEx(Object key, Object val) throws Exception { + if(containsKey(key)) throw new Exception("Key already present"); + return assoc(key, val); + } + + public IPersistentCollection cons(Object o) { + if (o instanceof Map.Entry) { + Map.Entry e = (Map.Entry) o; + return assoc(e.getKey(), e.getValue()); + } else if (o instanceof IPersistentVector) { + IPersistentVector v = (IPersistentVector) o; + if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); + return assoc(v.nth(0), v.nth(1)); + } + + DynamicMessage.Builder builder = builder(); + + for(ISeq s = RT.seq(o); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + Keyword key = (Keyword) e.getKey(); + setField(builder, fieldDescriptor(key), e.getValue()); + } + return makeNew(meta(), type, null, builder); + } + + public IPersistentMap without(Object key) throws Exception { + Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + if (field == null) return this; + if (field.isRequired()) throw new Exception("Can't remove required field"); + + DynamicMessage.Builder builder = builder(); + builder.clearField(field); + return makeNew(meta(), type, null, builder); + } + + public Iterator iterator() { + return new SeqIterator(seq()); + } + + public int count() { + return message().getAllFields().size(); + } + + public ISeq seq() { + return new Seq(null, message()); + } + + public IPersistentCollection empty() { + DynamicMessage.Builder builder = builder(); + builder.clear(); + return makeNew(meta(), type, null, builder); + } + + static class Seq extends ASeq { + final Map map; + final Descriptors.FieldDescriptor[] fields; + final int i; + + public Seq(IPersistentMap meta, DynamicMessage message) { + super(meta); + this.map = message.getAllFields(); + this.fields = (Descriptors.FieldDescriptor[])map.keySet().toArray(new Descriptors.FieldDescriptor[map.size()]); + this.i = 0; + } + + protected Seq(IPersistentMap meta, Map map, Descriptors.FieldDescriptor[] fields, int i){ + super(meta); + this.map = map; + this.fields = fields; + this.i = i; + } + + public Obj withMeta(IPersistentMap meta) { + if(meta != meta()) return new Seq(meta, map, fields, i); + return this; + } + + public Object first() { + Descriptors.FieldDescriptor field = fields[i]; + Keyword key = Keyword.intern(Symbol.intern(field.getName())); + return new MapEntry(key, map.get(field)); + } + + public ISeq next() { + if (i + 1 < fields.length) return new Seq(meta(), map, fields, i + 1); + return null; + } + } +} \ No newline at end of file From 429785712d46d3274cc4e859bac99d23db98ce09 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 19 Jan 2010 20:10:32 -0800 Subject: [PATCH 003/525] getting closer --- src/clj/protobuf.clj | 18 +- .../protobuf/PersistentProtocolBufferMap.java | 223 ++++++++++++------ 2 files changed, 158 insertions(+), 83 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index f6a0042..fdbeabd 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,10 +1,16 @@ (ns protobuf) +(set! *warn-on-reflection* true) + (defmacro defprotobuf [sym & args] - (let [class (symbol (apply str (interpose "$" (map name args))))] - `(def ~sym (. ~class getDescriptor)))) + (let [class (apply str (interpose "$" (map name args)))] + `(def ~sym (clojure.protobuf.PersistentProtocolBufferMap$Def/create ~class)))) + +(defn protobuf + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] + (clojure.protobuf.PersistentProtocolBufferMap/create type data)) + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] + (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply hash-map k v kvs)))) -(defn protobuf [desc & args] - (if (= (count args) 1) - (clojure.protobuf.PersistentProtocolBufferMap/create desc (first args)) - (clojure.protobuf.PersistentProtocolBufferMap/construct desc (apply hash-map args)))) \ No newline at end of file +(defn dump-protobuf [#^clojure.protobuf.PersistentProtocolBufferMap p] + (.toByteArray p)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index b91ffdd..503144a 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -15,56 +15,109 @@ import clojure.lang.*; import java.util.*; import java.io.InputStream; +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.lang.reflect.InvocationTargetException; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Descriptors; public class PersistentProtocolBufferMap extends APersistentMap { - final Descriptors.Descriptor type; + public static class Def { + final Descriptors.Descriptor type; + ConcurrentHashMap keyword_to_field; + static ConcurrentHashMap type_to_def = new ConcurrentHashMap(); + + public static Def create(String class_name) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, ClassNotFoundException { + Class c = Class.forName(class_name); + Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); + return create(type); + } + + public static Def create(Descriptors.Descriptor type) { + Def def = type_to_def.get(type); + if (def == null) { + def = new Def(type); + type_to_def.putIfAbsent(type, def); + } + return def; + } + + protected Def(Descriptors.Descriptor type) { + this.type = type; + this.keyword_to_field = new ConcurrentHashMap(); + } + + public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferException { + return DynamicMessage.parseFrom(type, bytes); + } + + public DynamicMessage.Builder newBuilder() { + return DynamicMessage.newBuilder(type); + } + + public Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { + Descriptors.FieldDescriptor field = keyword_to_field.get(key); + if (field == null) { + field = type.findFieldByName(key.getName()); + if (field != null) keyword_to_field.putIfAbsent(key, field); + } + return field; + } + + public String getName() { + return type.getName(); + } + + public String getFullName() { + return type.getFullName(); + } + + public Descriptors.Descriptor getMessageType() { + return type; + } + } + + final Def def; final DynamicMessage message; final DynamicMessage.Builder builder; - static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, byte[] bytes) - throws com.google.protobuf.InvalidProtocolBufferException - { - DynamicMessage message = DynamicMessage.parseFrom(type, bytes); - return new PersistentProtocolBufferMap(null, type, message, null); - } - - static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, String string) - throws com.google.protobuf.InvalidProtocolBufferException - { - return create(type, string.getBytes()); - } - - static public PersistentProtocolBufferMap create(Descriptors.Descriptor type, InputStream input) - throws com.google.protobuf.InvalidProtocolBufferException, java.io.IOException - { - DynamicMessage message = DynamicMessage.parseFrom(type, input); - return new PersistentProtocolBufferMap(null, type, message, null); + DynamicMessage built_message; + + static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws InvalidProtocolBufferException { + DynamicMessage message = def.parseFrom(bytes); + return new PersistentProtocolBufferMap(null, def, message, null); } - - static public PersistentProtocolBufferMap construct(Descriptors.Descriptor type, IPersistentMap keyvals){ - DynamicMessage.Builder builder = DynamicMessage.newBuilder(type); - PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, type, null, builder); + + static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { + DynamicMessage.Builder builder = def.newBuilder(); + PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def, null, builder); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); } - protected PersistentProtocolBufferMap(IPersistentMap meta, Descriptors.Descriptor type, - DynamicMessage message, DynamicMessage.Builder builder) { + protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { super(meta); - this.type = type; + this.def = def; this.message = message; this.builder = builder; } - protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Descriptors.Descriptor type, - DynamicMessage message, DynamicMessage.Builder builder) { - return new PersistentProtocolBufferMap(meta, type, message, builder); + protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { + return new PersistentProtocolBufferMap(meta, def, message, builder); } + public byte[] toByteArray() { + return message().toByteArray(); + } + + public Descriptors.Descriptor getMessageType() { + return def.getMessageType(); + } + protected DynamicMessage message() { if (message == null) { - return builder.build(); + if (built_message == null) built_message = builder.build(); + return built_message; } else { return message; } @@ -77,39 +130,68 @@ protected DynamicMessage.Builder builder() { return builder.clone(); } } - - protected Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { - return type.findFieldByName(key.getName()); + + static ConcurrentHashMap enum_to_keyword = new ConcurrentHashMap(); + + static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor e) { + Keyword k = enum_to_keyword.get(e); + if (k == null) { + k = Keyword.intern(Symbol.intern(e.getName().toLowerCase())); + } + return k; } - - protected Object fromFieldValue(Descriptors.FieldDescriptor.JavaType type, Object value) { - switch (type) { - case ENUM: - break; - case MESSAGE: - break; - default: - return value; + + static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { + if (value instanceof List) { + List values = (List) value; + + List items = new ArrayList(values.size()); + Iterator iterator = values.iterator(); + + while (iterator.hasNext()) { + items.add(fromProtoValue(field, iterator.next())); + } + return PersistentVector.create(items); + } else { + switch (field.getJavaType()) { + case ENUM: + Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor) value; + return enumToKeyword(e); + case MESSAGE: + + break; + default: + return value; + } + return null; } - return null; } - protected Object toFieldValue(Descriptors.FieldDescriptor.JavaType type, Object value) { - switch (type) { + static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { + switch (field.getJavaType()) { case LONG: if (value instanceof Long) return value; Integer i = (Integer) value; return new Long(i.longValue()); case INT: - return (Integer)value; + if (value instanceof Integer) return value; + Long l = (Long) value; + return new Integer(l.intValue()); case ENUM: - break; + Descriptors.EnumDescriptor e = field.getEnumType(); + Keyword key = (Keyword) value; + return e.findValueByName(key.getName().toUpperCase()); case MESSAGE: - break; + Descriptors.Descriptor d = field.getMessageType(); + if (value instanceof PersistentProtocolBufferMap) { + PersistentProtocolBufferMap m = (PersistentProtocolBufferMap) value; + if (m.getMessageType() == d) return m.message(); + } + Def def = PersistentProtocolBufferMap.Def.create(d); + return PersistentProtocolBufferMap.construct(def, (IPersistentMap) value); default: return value; } - return null; } protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { @@ -118,22 +200,22 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip if (field.isRepeated()) { builder.clearField(field); for (ISeq s = RT.seq(val); s != null; s = s.next()) { - Object value = toFieldValue(field.getJavaType(), s.first()); + Object value = toProtoValue(field, s.first()); builder.addRepeatedField(field, value); } } else { - Object value = toFieldValue(field.getJavaType(), val); + Object value = toProtoValue(field, val); builder.setField(field, value); } } public Obj withMeta(IPersistentMap meta) { if (meta == meta()) return this; - return makeNew(meta, type, message, builder); + return makeNew(meta, def, message, builder); } public boolean containsKey(Object key) { - Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); return message().hasField(field); } @@ -143,23 +225,9 @@ public IMapEntry entryAt(Object key) { } public Object valAt(Object key) { - Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return null; - - DynamicMessage message = message(); - - if (field.isRepeated()) { - int num_items = message.getRepeatedFieldCount(field); - List items = new ArrayList(num_items); - - for (int i = 0; i < num_items; i++) { - Object value = fromFieldValue(field.getJavaType(), message.getRepeatedField(field, i)); - items.add(value); - } - return PersistentVector.create(items); - } else { - return fromFieldValue(field.getJavaType(), message.getField(field)); - } + return fromProtoValue(field, message().getField(field)); } public Object valAt(Object key, Object notFound) { @@ -169,11 +237,11 @@ public Object valAt(Object key, Object notFound) { public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); - Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return this; setField(builder, field, val); - return makeNew(meta(), type, null, builder); + return makeNew(meta(), def, null, builder); } public IPersistentMap assocEx(Object key, Object val) throws Exception { @@ -196,19 +264,19 @@ public IPersistentCollection cons(Object o) { for(ISeq s = RT.seq(o); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Keyword key = (Keyword) e.getKey(); - setField(builder, fieldDescriptor(key), e.getValue()); + setField(builder, def.fieldDescriptor(key), e.getValue()); } - return makeNew(meta(), type, null, builder); + return makeNew(meta(), def, null, builder); } public IPersistentMap without(Object key) throws Exception { - Descriptors.FieldDescriptor field = fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return this; if (field.isRequired()) throw new Exception("Can't remove required field"); DynamicMessage.Builder builder = builder(); builder.clearField(field); - return makeNew(meta(), type, null, builder); + return makeNew(meta(), def, null, builder); } public Iterator iterator() { @@ -226,7 +294,7 @@ public ISeq seq() { public IPersistentCollection empty() { DynamicMessage.Builder builder = builder(); builder.clear(); - return makeNew(meta(), type, null, builder); + return makeNew(meta(), def, null, builder); } static class Seq extends ASeq { @@ -256,7 +324,8 @@ public Obj withMeta(IPersistentMap meta) { public Object first() { Descriptors.FieldDescriptor field = fields[i]; Keyword key = Keyword.intern(Symbol.intern(field.getName())); - return new MapEntry(key, map.get(field)); + Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); + return new MapEntry(key, val); } public ISeq next() { From b5f865e7fa468f25a6c8f6b3f6a8c5ab3ce6a1b6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 20 Jan 2010 10:08:26 -0800 Subject: [PATCH 004/525] add support for nested messages --- .../protobuf/PersistentProtocolBufferMap.java | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 503144a..bc85d8d 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -8,8 +8,6 @@ * You must not remove this notice, or any other, from this software. **/ -/* Based on rhickey's clojure.lang.PersistentStructMap - Jan 17, 2010 */ - package clojure.protobuf; import clojure.lang.*; @@ -102,7 +100,7 @@ protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessa this.builder = builder; } - protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { + static protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { return new PersistentProtocolBufferMap(meta, def, message, builder); } @@ -158,12 +156,12 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor) value; return enumToKeyword(e); case MESSAGE: - - break; + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + DynamicMessage message = (DynamicMessage) value; + return PersistentProtocolBufferMap.makeNew(null, def, message, null); default: return value; } - return null; } } @@ -182,13 +180,14 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v Keyword key = (Keyword) value; return e.findValueByName(key.getName().toUpperCase()); case MESSAGE: - Descriptors.Descriptor d = field.getMessageType(); + PersistentProtocolBufferMap protobuf; if (value instanceof PersistentProtocolBufferMap) { - PersistentProtocolBufferMap m = (PersistentProtocolBufferMap) value; - if (m.getMessageType() == d) return m.message(); + protobuf = (PersistentProtocolBufferMap) value; + } else { + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + protobuf = PersistentProtocolBufferMap.construct(def, (IPersistentMap) value); } - Def def = PersistentProtocolBufferMap.Def.create(d); - return PersistentProtocolBufferMap.construct(def, (IPersistentMap) value); + return protobuf.message(); default: return value; } @@ -288,7 +287,7 @@ public int count() { } public ISeq seq() { - return new Seq(null, message()); + return Seq.create(message()); } public IPersistentCollection empty() { @@ -302,11 +301,13 @@ static class Seq extends ASeq { final Descriptors.FieldDescriptor[] fields; final int i; - public Seq(IPersistentMap meta, DynamicMessage message) { - super(meta); - this.map = message.getAllFields(); - this.fields = (Descriptors.FieldDescriptor[])map.keySet().toArray(new Descriptors.FieldDescriptor[map.size()]); - this.i = 0; + static public Seq create(DynamicMessage message) { + Map map = message.getAllFields(); + if (map.size() == 0) return null; + + Descriptors.FieldDescriptor[] fields = new Descriptors.FieldDescriptor[map.size()]; + fields = (Descriptors.FieldDescriptor[]) map.keySet().toArray(fields); + return new Seq(null, map, fields, 0); } protected Seq(IPersistentMap meta, Map map, Descriptors.FieldDescriptor[] fields, int i){ @@ -322,6 +323,7 @@ public Obj withMeta(IPersistentMap meta) { } public Object first() { + if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; Keyword key = Keyword.intern(Symbol.intern(field.getName())); Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); From a5a3a4bc4ed14688b44fb8c5bbf46d07b6cec5f2 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 20 Jan 2010 19:00:25 -0800 Subject: [PATCH 005/525] improvements and bugfixes to support nested messages and enums --- src/clj/protobuf.clj | 5 +- .../protobuf/PersistentProtocolBufferMap.java | 86 ++++++++++++------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index fdbeabd..5145d46 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -6,11 +6,14 @@ (let [class (apply str (interpose "$" (map name args)))] `(def ~sym (clojure.protobuf.PersistentProtocolBufferMap$Def/create ~class)))) -(defn protobuf +(defn protobuf + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] + (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] (clojure.protobuf.PersistentProtocolBufferMap/create type data)) ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply hash-map k v kvs)))) + (defn dump-protobuf [#^clojure.protobuf.PersistentProtocolBufferMap p] (.toByteArray p)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index bc85d8d..26a6953 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -13,7 +13,8 @@ import clojure.lang.*; import java.util.*; import java.io.InputStream; -import java.io.IOException; +import java.io.PrintWriter; +import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.lang.reflect.InvocationTargetException; import com.google.protobuf.InvalidProtocolBufferException; @@ -55,9 +56,11 @@ public DynamicMessage.Builder newBuilder() { } public Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { + if (key == null) return null; Descriptors.FieldDescriptor field = keyword_to_field.get(key); if (field == null) { - field = type.findFieldByName(key.getName()); + String name = key.getName().replaceAll("-","_"); + field = type.findFieldByName(name); if (field != null) keyword_to_field.putIfAbsent(key, field); } return field; @@ -76,32 +79,37 @@ public Descriptors.Descriptor getMessageType() { } } - final Def def; - final DynamicMessage message; - final DynamicMessage.Builder builder; + final Def def; + final DynamicMessage message; DynamicMessage built_message; static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws InvalidProtocolBufferException { DynamicMessage message = def.parseFrom(bytes); - return new PersistentProtocolBufferMap(null, def, message, null); + return new PersistentProtocolBufferMap(null, def, message); } static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { - DynamicMessage.Builder builder = def.newBuilder(); - PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def, null, builder); + PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); } - protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { + protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { + super(meta); + this.def = def; + this.message = null; + } + + protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message) { super(meta); this.def = def; this.message = message; - this.builder = builder; } - - static protected PersistentProtocolBufferMap makeNew(IPersistentMap meta, Def def, DynamicMessage message, DynamicMessage.Builder builder) { - return new PersistentProtocolBufferMap(meta, def, message, builder); + + protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage.Builder builder) { + super(meta); + this.def = def; + this.message = builder.build(); } public byte[] toByteArray() { @@ -114,29 +122,30 @@ public Descriptors.Descriptor getMessageType() { protected DynamicMessage message() { if (message == null) { - if (built_message == null) built_message = builder.build(); - return built_message; + return def.newBuilder().build(); // This will only work if an empty message is valid. } else { return message; } } protected DynamicMessage.Builder builder() { - if (builder == null) { - return message.toBuilder(); + if (message == null) { + return def.newBuilder(); } else { - return builder.clone(); + return message.toBuilder(); } } static ConcurrentHashMap enum_to_keyword = new ConcurrentHashMap(); - static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor e) { - Keyword k = enum_to_keyword.get(e); - if (k == null) { - k = Keyword.intern(Symbol.intern(e.getName().toLowerCase())); + static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { + Keyword keyword = enum_to_keyword.get(enum_value); + if (keyword == null) { + String name = enum_value.getName().toLowerCase().replaceAll("_","-"); + keyword = Keyword.intern(Symbol.intern(name)); + enum_to_keyword.putIfAbsent(enum_value, keyword); } - return k; + return keyword; } static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { @@ -158,7 +167,11 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object case MESSAGE: Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); DynamicMessage message = (DynamicMessage) value; - return PersistentProtocolBufferMap.makeNew(null, def, message, null); + + // Total hack because getField() doesn't return an empty array for repeated messages. + if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList()); + + return new PersistentProtocolBufferMap(null, def, message); default: return value; } @@ -176,9 +189,15 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v Long l = (Long) value; return new Integer(l.intValue()); case ENUM: - Descriptors.EnumDescriptor e = field.getEnumType(); Keyword key = (Keyword) value; - return e.findValueByName(key.getName().toUpperCase()); + String name = key.getName().toUpperCase().replaceAll("-","_"); + Descriptors.EnumDescriptor enum_type = field.getEnumType(); + Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); + if (enum_value == null) { + PrintWriter err = (PrintWriter) RT.ERR.deref(); + err.format("invalid enum value %s for enum type %s\n", name, enum_type.getFullName()); + } + return enum_value; case MESSAGE: PersistentProtocolBufferMap protobuf; if (value instanceof PersistentProtocolBufferMap) { @@ -210,7 +229,7 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip public Obj withMeta(IPersistentMap meta) { if (meta == meta()) return this; - return makeNew(meta, def, message, builder); + return new PersistentProtocolBufferMap(meta(), def, message); } public boolean containsKey(Object key) { @@ -234,13 +253,13 @@ public Object valAt(Object key, Object notFound) { return (val == null) ? notFound : val; } - public IPersistentMap assoc(Object key, Object val) { + public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return this; setField(builder, field, val); - return makeNew(meta(), def, null, builder); + return new PersistentProtocolBufferMap(meta(), def, builder); } public IPersistentMap assocEx(Object key, Object val) throws Exception { @@ -265,7 +284,7 @@ public IPersistentCollection cons(Object o) { Keyword key = (Keyword) e.getKey(); setField(builder, def.fieldDescriptor(key), e.getValue()); } - return makeNew(meta(), def, null, builder); + return new PersistentProtocolBufferMap(meta(), def, builder); } public IPersistentMap without(Object key) throws Exception { @@ -275,7 +294,7 @@ public IPersistentMap without(Object key) throws Exception { DynamicMessage.Builder builder = builder(); builder.clearField(field); - return makeNew(meta(), def, null, builder); + return new PersistentProtocolBufferMap(meta(), def, builder); } public Iterator iterator() { @@ -293,7 +312,7 @@ public ISeq seq() { public IPersistentCollection empty() { DynamicMessage.Builder builder = builder(); builder.clear(); - return makeNew(meta(), def, null, builder); + return new PersistentProtocolBufferMap(meta(), def, builder); } static class Seq extends ASeq { @@ -325,7 +344,8 @@ public Obj withMeta(IPersistentMap meta) { public Object first() { if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; - Keyword key = Keyword.intern(Symbol.intern(field.getName())); + String name = field.getName().replaceAll("_","-"); + Keyword key = Keyword.intern(Symbol.intern(name)); Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); return new MapEntry(key, val); } From 7cb5b83b067800333f646795d23cd026040ceb2b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 27 Jan 2010 17:54:13 -0800 Subject: [PATCH 006/525] add ant tasks for building --- .gitignore | 3 +- build.xml | 66 ++++++++++++++++++++++++++++++++++++++++++++ src/clj/protobuf.clj | 5 ++-- 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 build.xml diff --git a/.gitignore b/.gitignore index cb290e3..e666a96 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -*.class +lib +classes *.jar \ No newline at end of file diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..257ded2 --- /dev/null +++ b/build.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 5145d46..b400743 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -10,10 +10,9 @@ ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] - (clojure.protobuf.PersistentProtocolBufferMap/create type data)) + (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data))) ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply hash-map k v kvs)))) - -(defn dump-protobuf [#^clojure.protobuf.PersistentProtocolBufferMap p] +(defn protobuf-bytes [#^clojure.protobuf.PersistentProtocolBufferMap p] (.toByteArray p)) \ No newline at end of file From 5923d6179699a39fc997995b80dfe5a625910099 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 27 Jan 2010 18:23:31 -0800 Subject: [PATCH 007/525] add readme and example --- README | 0 README.md | 45 ++++++++++++++++++++++++++++++++++++++++++ examples/.gitignore | 2 ++ examples/example.proto | 5 +++++ 4 files changed, 52 insertions(+) delete mode 100644 README create mode 100644 README.md create mode 100644 examples/.gitignore create mode 100644 examples/example.proto diff --git a/README b/README deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f4023c --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +Clojure-protobuf provides a clojure interface to Google's protocol buffers (http://code.google.com/p/protobuf). + +## Build + + ant + +## Usage + +Write a `.proto` file: + + message Person { + required int32 id = 1; + required string name = 2; + optional string email = 3; + repeated string nicknames = 4; + } + +Compile the file to Java: + + protoc --java_out=. example.proto + +Now you can use the protocol buffer in clojure: + + (use 'protobuf) + (defprotobuf person Example Person) + + (def p (protobuf person :id 4 :name "Bob" :email "bob@example.com")) + => {:id 4, :name "Bob", :email "bob@example.com"} + + (assoc p :name "Bill")) + => {:id 4, :name "Bill", :email "bob@example.com"} + + (def b (protobuf-bytes p)) + => # + + (protobuf person b) + => {:id 4, :name "Bob", :email "bob@example.com"} + +A protocol buffer map is immutable just like other clojure objects. It is similar to a +struct-map, except you cannot insert fields that aren't specified in the `.proto` file. + +## Motivation + +So why would you want to do this? Because it is WAY faster to serialize and deserialize +protocol buffers than standard clojure objects. diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..6a70a53 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,2 @@ +*.class +*.java \ No newline at end of file diff --git a/examples/example.proto b/examples/example.proto new file mode 100644 index 0000000..56b85e2 --- /dev/null +++ b/examples/example.proto @@ -0,0 +1,5 @@ +message Person { + required int32 id = 1; + required string name = 2; + optional string email = 3; +} From 19a12bc426881da45da01864b75b3f5465170731 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 27 Jan 2010 18:24:51 -0800 Subject: [PATCH 008/525] update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7f4023c..f684fa0 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,5 @@ Clojure-protobuf provides a clojure interface to Google's protocol buffers (http://code.google.com/p/protobuf). -## Build - - ant - ## Usage Write a `.proto` file: @@ -43,3 +39,7 @@ struct-map, except you cannot insert fields that aren't specified in the `.proto So why would you want to do this? Because it is WAY faster to serialize and deserialize protocol buffers than standard clojure objects. + +## Build + + ant package \ No newline at end of file From 7109d585cfe23c7fee5128a9f3895b7bcd5fb61b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 27 Jan 2010 18:25:50 -0800 Subject: [PATCH 009/525] update readme --- README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f684fa0..02ff81e 100644 --- a/README.md +++ b/README.md @@ -4,33 +4,33 @@ Clojure-protobuf provides a clojure interface to Google's protocol buffers (http Write a `.proto` file: - message Person { - required int32 id = 1; - required string name = 2; - optional string email = 3; - repeated string nicknames = 4; - } + message Person { + required int32 id = 1; + required string name = 2; + optional string email = 3; + repeated string nicknames = 4; + } Compile the file to Java: - protoc --java_out=. example.proto + protoc --java_out=. example.proto Now you can use the protocol buffer in clojure: - (use 'protobuf) - (defprotobuf person Example Person) + (use 'protobuf) + (defprotobuf person Example Person) - (def p (protobuf person :id 4 :name "Bob" :email "bob@example.com")) - => {:id 4, :name "Bob", :email "bob@example.com"} + (def p (protobuf person :id 4 :name "Bob" :email "bob@example.com")) + => {:id 4, :name "Bob", :email "bob@example.com"} - (assoc p :name "Bill")) - => {:id 4, :name "Bill", :email "bob@example.com"} + (assoc p :name "Bill")) + => {:id 4, :name "Bill", :email "bob@example.com"} - (def b (protobuf-bytes p)) - => # + (def b (protobuf-bytes p)) + => # - (protobuf person b) - => {:id 4, :name "Bob", :email "bob@example.com"} + (protobuf person b) + => {:id 4, :name "Bob", :email "bob@example.com"} A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. @@ -42,4 +42,4 @@ protocol buffers than standard clojure objects. ## Build - ant package \ No newline at end of file + ant package \ No newline at end of file From 1b01173dc3cc09c5406ab851479bcf5d32940738 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 27 Jan 2010 18:27:49 -0800 Subject: [PATCH 010/525] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 02ff81e..614c9d0 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ struct-map, except you cannot insert fields that aren't specified in the `.proto ## Motivation So why would you want to do this? Because it is WAY faster to serialize and deserialize -protocol buffers than standard clojure objects. +protocol buffers than standard clojure objects. Protocol buffers are also a great way to +communicate with other languages over the network. ## Build From 341a12f5ec469abb6fe73dc20e751c36eb1144ae Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 28 Jan 2010 16:18:42 -0800 Subject: [PATCH 011/525] fix ant to fail if clojure.jar is not specified --- README.md | 10 +++------- build.xml | 33 ++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 614c9d0..1041b3d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ Clojure-protobuf provides a clojure interface to Google's protocol buffers (http://code.google.com/p/protobuf). +Protocol buffers can be used to communicate with other languages over the network, and +they are WAY faster to serialize and deserialize than standard Clojure objects. ## Usage @@ -35,12 +37,6 @@ Now you can use the protocol buffer in clojure: A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. -## Motivation - -So why would you want to do this? Because it is WAY faster to serialize and deserialize -protocol buffers than standard clojure objects. Protocol buffers are also a great way to -communicate with other languages over the network. - ## Build - ant package \ No newline at end of file + ant package -Dclojure.jar= \ No newline at end of file diff --git a/build.xml b/build.xml index 257ded2..73c59de 100644 --- a/build.xml +++ b/build.xml @@ -19,29 +19,18 @@ - - + + - - - - - - - - - - - - + @@ -51,7 +40,21 @@ - + + + + You have not defined a path to clojure.jar so I can't compile files. + To enable compiling, run "ant -Dclojure.jar=<...path to clojure.jar..>" + + + + + + + + + + From 8a65e8c8d983c1e9d0c31fe8584610198f5d7a1c Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 29 Jan 2010 14:31:36 -0800 Subject: [PATCH 012/525] improved installation --- README.md | 15 +++++++++++-- build.xml | 65 ++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1041b3d..2a2df9b 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,17 @@ Now you can use the protocol buffer in clojure: A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. -## Build +## Installation - ant package -Dclojure.jar= \ No newline at end of file +To download clojure.jar and google's protobuf source automatically and install protoc: + ant package + ant install + +You can specify options to configure (like the install prefix): + ant package -Dconfigure='--prefix=/opt/local' + ant install + +You can also specify a specific location for clojure.jar or the protobuf source: + ant package -Dclojure.jar=$HOME/lib/java/clojure-1.1.0.jar -Dprotobuf=$HOME/Downloads/protobuf-2.3.0 + +This code has been tested with clojure version 1.1.0 and protobuf version 2.3.0. diff --git a/build.xml b/build.xml index 73c59de..738bc87 100644 --- a/build.xml +++ b/build.xml @@ -2,50 +2,75 @@ - + + + + + + - - - - + - + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - You have not defined a path to clojure.jar so I can't compile files. - To enable compiling, run "ant -Dclojure.jar=<...path to clojure.jar..>" - + + + + + @@ -54,9 +79,9 @@ - + - + From 901c2ca299ad3fa05c3b2374c603f91fe3b781a5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 29 Jan 2010 14:33:47 -0800 Subject: [PATCH 013/525] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2a2df9b..6511666 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,11 @@ To download clojure.jar and google's protobuf source automatically and install p ant package ant install -You can specify options to configure (like the install prefix): +You can also specify options to configure (like the install prefix): ant package -Dconfigure='--prefix=/opt/local' ant install -You can also specify a specific location for clojure.jar or the protobuf source: +Or you can specify a specific location for clojure.jar or the protobuf source: ant package -Dclojure.jar=$HOME/lib/java/clojure-1.1.0.jar -Dprotobuf=$HOME/Downloads/protobuf-2.3.0 This code has been tested with clojure version 1.1.0 and protobuf version 2.3.0. From 3b834050b2188250037a1c56f5ccd96b18d5bcf6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 1 Feb 2010 13:54:03 -0800 Subject: [PATCH 014/525] improved build.xml --- build.xml | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/build.xml b/build.xml index 738bc87..17554cd 100644 --- a/build.xml +++ b/build.xml @@ -1,12 +1,16 @@ + + + + - - - + + + @@ -21,8 +25,8 @@ - - + + @@ -30,28 +34,21 @@ - + - - - - - - - - - - + + + - + - + @@ -65,8 +62,8 @@ - - + + From 7bee6936f0eb6aa3522c0279df00f2986ecb0ed6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 3 Feb 2010 21:23:56 -0800 Subject: [PATCH 015/525] stage install into lib/install --- build.xml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/build.xml b/build.xml index 17554cd..c9ce185 100644 --- a/build.xml +++ b/build.xml @@ -8,15 +8,17 @@ - - - + + + + + @@ -34,22 +36,27 @@ - + + - + + + + + - - - + + + From 62f7f9387a250e987ee2e7294efd6c54108f9ddd Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 4 Feb 2010 02:42:31 -0800 Subject: [PATCH 016/525] fail on install error --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index c9ce185..ebeb9e6 100644 --- a/build.xml +++ b/build.xml @@ -55,7 +55,7 @@ - + From 6dc5cce3ee782e3c192032838d60fb78524d2611 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 9 Feb 2010 11:52:40 -0800 Subject: [PATCH 017/525] add protobuf? and protodef? methods --- src/clj/protobuf.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index b400743..4157089 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -6,6 +6,12 @@ (let [class (apply str (interpose "$" (map name args)))] `(def ~sym (clojure.protobuf.PersistentProtocolBufferMap$Def/create ~class)))) +(defn protobuf? [obj] + (instance? clojure.protobuf.PersistentProtocolBufferMap obj)) + +(defn protodef? [obj] + (instance? clojure.protobuf.PersistentProtocolBufferMap$Def obj)) + (defn protobuf ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) From c30d542f0f0ab1e9e625a2035f60fd6d79df651e Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Feb 2010 13:05:39 -0800 Subject: [PATCH 018/525] add protodef method --- src/clj/protobuf.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 4157089..46bfe2a 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -2,9 +2,12 @@ (set! *warn-on-reflection* true) +(defn protodef [class] + (clojure.protobuf.PersistentProtocolBufferMap$Def/create class)) + (defmacro defprotobuf [sym & args] (let [class (apply str (interpose "$" (map name args)))] - `(def ~sym (clojure.protobuf.PersistentProtocolBufferMap$Def/create ~class)))) + `(def ~sym (protodef ~class)))) (defn protobuf? [obj] (instance? clojure.protobuf.PersistentProtocolBufferMap obj)) From fe4be5d11bec70c404f9c64a45f9e15280b93630 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Feb 2010 13:06:31 -0800 Subject: [PATCH 019/525] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6511666..2e235ce 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Clojure-protobuf provides a clojure interface to Google's protocol buffers (http://code.google.com/p/protobuf). +Clojure-protobuf provides a clojure interface to Google's [protocol buffers](http://code.google.com/p/protobuf). Protocol buffers can be used to communicate with other languages over the network, and they are WAY faster to serialize and deserialize than standard Clojure objects. From 75fc03718e3ec7382917a3341fa539103b54acab Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Feb 2010 14:07:41 -0800 Subject: [PATCH 020/525] support protodef creation with string or class --- .../protobuf/PersistentProtocolBufferMap.java | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 26a6953..8b714b9 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -26,9 +26,13 @@ public static class Def { final Descriptors.Descriptor type; ConcurrentHashMap keyword_to_field; static ConcurrentHashMap type_to_def = new ConcurrentHashMap(); - + public static Def create(String class_name) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, ClassNotFoundException { Class c = Class.forName(class_name); + return create(c); + } + + public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); return create(type); } @@ -49,7 +53,7 @@ protected Def(Descriptors.Descriptor type) { public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferException { return DynamicMessage.parseFrom(type, bytes); - } + } public DynamicMessage.Builder newBuilder() { return DynamicMessage.newBuilder(type); @@ -65,7 +69,7 @@ public Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { } return field; } - + public String getName() { return type.getName(); } @@ -88,12 +92,12 @@ static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws I DynamicMessage message = def.parseFrom(bytes); return new PersistentProtocolBufferMap(null, def, message); } - + static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); } - + protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { super(meta); this.def = def; @@ -111,7 +115,7 @@ protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessa this.def = def; this.message = builder.build(); } - + public byte[] toByteArray() { return message().toByteArray(); } @@ -127,7 +131,7 @@ protected DynamicMessage message() { return message; } } - + protected DynamicMessage.Builder builder() { if (message == null) { return def.newBuilder(); @@ -154,7 +158,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object List items = new ArrayList(values.size()); Iterator iterator = values.iterator(); - + while (iterator.hasNext()) { items.add(fromProtoValue(field, iterator.next())); } @@ -177,17 +181,17 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } } - + static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { switch (field.getJavaType()) { case LONG: if (value instanceof Long) return value; Integer i = (Integer) value; - return new Long(i.longValue()); + return new Long(i.longValue()); case INT: if (value instanceof Integer) return value; Long l = (Long) value; - return new Integer(l.intValue()); + return new Integer(l.intValue()); case ENUM: Keyword key = (Keyword) value; String name = key.getName().toUpperCase().replaceAll("-","_"); @@ -211,7 +215,7 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v return value; } } - + protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { if (field == null) return; @@ -226,47 +230,47 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip builder.setField(field, value); } } - + public Obj withMeta(IPersistentMap meta) { if (meta == meta()) return this; return new PersistentProtocolBufferMap(meta(), def, message); } - + public boolean containsKey(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); return message().hasField(field); } - + public IMapEntry entryAt(Object key) { Object value = valAt(key); return (value == null) ? null : new MapEntry(key, value); } - + public Object valAt(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return null; return fromProtoValue(field, message().getField(field)); } - + public Object valAt(Object key, Object notFound) { Object val = valAt(key); return (val == null) ? notFound : val; } - - public IPersistentMap assoc(Object key, Object val) { + + public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); - + if (field == null) return this; setField(builder, field, val); return new PersistentProtocolBufferMap(meta(), def, builder); } - + public IPersistentMap assocEx(Object key, Object val) throws Exception { if(containsKey(key)) throw new Exception("Key already present"); return assoc(key, val); } - + public IPersistentCollection cons(Object o) { if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry) o; @@ -276,9 +280,9 @@ public IPersistentCollection cons(Object o) { if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); return assoc(v.nth(0), v.nth(1)); } - + DynamicMessage.Builder builder = builder(); - + for(ISeq s = RT.seq(o); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Keyword key = (Keyword) e.getKey(); @@ -286,40 +290,40 @@ public IPersistentCollection cons(Object o) { } return new PersistentProtocolBufferMap(meta(), def, builder); } - + public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); if (field == null) return this; if (field.isRequired()) throw new Exception("Can't remove required field"); - + DynamicMessage.Builder builder = builder(); builder.clearField(field); return new PersistentProtocolBufferMap(meta(), def, builder); } - + public Iterator iterator() { return new SeqIterator(seq()); } - + public int count() { return message().getAllFields().size(); } - + public ISeq seq() { return Seq.create(message()); } - + public IPersistentCollection empty() { DynamicMessage.Builder builder = builder(); builder.clear(); return new PersistentProtocolBufferMap(meta(), def, builder); } - + static class Seq extends ASeq { final Map map; final Descriptors.FieldDescriptor[] fields; final int i; - + static public Seq create(DynamicMessage message) { Map map = message.getAllFields(); if (map.size() == 0) return null; @@ -328,19 +332,19 @@ static public Seq create(DynamicMessage message) { fields = (Descriptors.FieldDescriptor[]) map.keySet().toArray(fields); return new Seq(null, map, fields, 0); } - + protected Seq(IPersistentMap meta, Map map, Descriptors.FieldDescriptor[] fields, int i){ super(meta); this.map = map; this.fields = fields; this.i = i; } - + public Obj withMeta(IPersistentMap meta) { if(meta != meta()) return new Seq(meta, map, fields, i); return this; } - + public Object first() { if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; @@ -349,7 +353,7 @@ public Object first() { Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); return new MapEntry(key, val); } - + public ISeq next() { if (i + 1 < fields.length) return new Seq(meta(), map, fields, i + 1); return null; From be55c4d33d796392f83bd17ae5f27b99cd1b7690 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Feb 2010 14:08:09 -0800 Subject: [PATCH 021/525] remove warn-on-reflection --- src/clj/protobuf.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 46bfe2a..3cb1f3f 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,7 +1,5 @@ (ns protobuf) -(set! *warn-on-reflection* true) - (defn protodef [class] (clojure.protobuf.PersistentProtocolBufferMap$Def/create class)) From 9c6afc7bb2be2781a6e7cfbcdb2429b9337936bd Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Feb 2010 17:15:31 -0800 Subject: [PATCH 022/525] fix function order --- src/clj/protobuf.clj | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 3cb1f3f..31a1df2 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,11 +1,6 @@ (ns protobuf) -(defn protodef [class] - (clojure.protobuf.PersistentProtocolBufferMap$Def/create class)) - -(defmacro defprotobuf [sym & args] - (let [class (apply str (interpose "$" (map name args)))] - `(def ~sym (protodef ~class)))) +;(set! *warn-on-reflection* true) (defn protobuf? [obj] (instance? clojure.protobuf.PersistentProtocolBufferMap obj)) @@ -13,6 +8,15 @@ (defn protodef? [obj] (instance? clojure.protobuf.PersistentProtocolBufferMap$Def obj)) +(defn protodef [class] + (if (or (protodef? class) (nil? class)) + class + (clojure.protobuf.PersistentProtocolBufferMap$Def/create class))) + +(defmacro defprotobuf [sym & args] + (let [class (apply str (interpose "$" (map name args)))] + `(def ~sym (protodef ~class)))) + (defn protobuf ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) From 959a127a173530405913154462a68126d14cf5a0 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 12 Mar 2010 16:35:27 -0800 Subject: [PATCH 023/525] make conj work the same as appending two protobufs, accept strings in the place of keywords for keys and enums, support loading truncated protobufs with CodedInputStream --- src/clj/protobuf.clj | 14 +- .../protobuf/PersistentProtocolBufferMap.java | 124 +++++++++++++----- 2 files changed, 100 insertions(+), 38 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 31a1df2..f940df9 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,7 +1,5 @@ (ns protobuf) -;(set! *warn-on-reflection* true) - (defn protobuf? [obj] (instance? clojure.protobuf.PersistentProtocolBufferMap obj)) @@ -20,10 +18,16 @@ (defn protobuf ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type m] + (clojure.protobuf.PersistentProtocolBufferMap/construct type m)) + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] + (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) + +(defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data))) - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] - (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply hash-map k v kvs)))) + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data #^Integer length] + (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data length)))) -(defn protobuf-bytes [#^clojure.protobuf.PersistentProtocolBufferMap p] +(defn protobuf-dump [#^clojure.protobuf.PersistentProtocolBufferMap p] (.toByteArray p)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 8b714b9..1e97ff7 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -20,8 +20,9 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Descriptors; +import com.google.protobuf.CodedInputStream; -public class PersistentProtocolBufferMap extends APersistentMap { +public class PersistentProtocolBufferMap extends APersistentMap { //implements IPersistentSet{ public static class Def { final Descriptors.Descriptor type; ConcurrentHashMap keyword_to_field; @@ -32,7 +33,7 @@ public static Def create(String class_name) throws NoSuchMethodException, Illega return create(c); } - public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); return create(type); } @@ -55,19 +56,29 @@ public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferExcept return DynamicMessage.parseFrom(type, bytes); } + public DynamicMessage parseFrom(CodedInputStream input) throws IOException { + return DynamicMessage.parseFrom(type, input); + } + public DynamicMessage.Builder newBuilder() { return DynamicMessage.newBuilder(type); } - public Descriptors.FieldDescriptor fieldDescriptor(Keyword key) { + public Descriptors.FieldDescriptor fieldDescriptor(Object key) { if (key == null) return null; - Descriptors.FieldDescriptor field = keyword_to_field.get(key); - if (field == null) { - String name = key.getName().replaceAll("-","_"); - field = type.findFieldByName(name); - if (field != null) keyword_to_field.putIfAbsent(key, field); + + if (key instanceof Keyword) { + Keyword keyword = (Keyword) key; + Descriptors.FieldDescriptor field = keyword_to_field.get(keyword); + if (field == null) { + field = fieldDescriptor(keyword.getName()); + if (field != null) keyword_to_field.putIfAbsent(keyword, field); + } + return field; + } else { + String name = ((String) key).replaceAll("-","_"); + return type.findFieldByName(name); } - return field; } public String getName() { @@ -93,6 +104,12 @@ static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws I return new PersistentProtocolBufferMap(null, def, message); } + static public PersistentProtocolBufferMap create(Def def, byte[] bytes, Integer length) throws IOException { + CodedInputStream input = CodedInputStream.newInstance(bytes, 0, length.intValue()); + DynamicMessage message = def.parseFrom(input); + return new PersistentProtocolBufferMap(null, def, message); + } + static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); @@ -155,10 +172,9 @@ static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_valu static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { if (value instanceof List) { List values = (List) value; - - List items = new ArrayList(values.size()); Iterator iterator = values.iterator(); + List items = new ArrayList(values.size()); while (iterator.hasNext()) { items.add(fromProtoValue(field, iterator.next())); } @@ -193,8 +209,8 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v Long l = (Long) value; return new Integer(l.intValue()); case ENUM: - Keyword key = (Keyword) value; - String name = key.getName().toUpperCase().replaceAll("-","_"); + String name = (value instanceof Keyword) ? ((Keyword) value).getName() : (String) value; + name = name.toUpperCase().replaceAll("-","_"); Descriptors.EnumDescriptor enum_type = field.getEnumType(); Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); if (enum_value == null) { @@ -216,13 +232,21 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v } } - protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { + protected void addField(DynamicMessage.Builder builder, Keyword key, Object val) { + addField(builder, def.fieldDescriptor(key), val); + } + + protected void addField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { if (field == null) return; if (field.isRepeated()) { - builder.clearField(field); - for (ISeq s = RT.seq(val); s != null; s = s.next()) { - Object value = toProtoValue(field, s.first()); + if (val instanceof Sequential) { + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Object value = toProtoValue(field, s.first()); + builder.addRepeatedField(field, value); + } + } else { + Object value = toProtoValue(field, val); builder.addRepeatedField(field, value); } } else { @@ -236,9 +260,42 @@ public Obj withMeta(IPersistentMap meta) { return new PersistentProtocolBufferMap(meta(), def, message); } + public boolean contains(Object key){ + if (key instanceof IPersistentMap) { + for (ISeq s = RT.seq(key); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + Object v = e.getValue(); + Object value = valAt(e.getKey()); + + if (value instanceof APersistentVector) { + APersistentVector vec = (APersistentVector) value; + if (!vec.contains(v)) return false; + } else if (value instanceof IPersistentSet) { + IPersistentSet set = (IPersistentSet) value; + if (!set.contains(v)) return false; + } else { + if (value != v) return false; + } + } + return true; + } else { + return false; + } + } + public boolean containsKey(Object key) { - Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); - return message().hasField(field); + if (key instanceof IPersistentMap) { + return contains(key); + } else { + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + if (field == null) { + return false; + } else if (field.isRepeated()) { + return message().getRepeatedFieldCount(field) > 0; + } else { + return message().hasField(field); + } + } } public IMapEntry entryAt(Object key) { @@ -247,7 +304,7 @@ public IMapEntry entryAt(Object key) { } public Object valAt(Object key) { - Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return null; return fromProtoValue(field, message().getField(field)); } @@ -259,10 +316,12 @@ public Object valAt(Object key, Object notFound) { public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); - Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return this; - setField(builder, field, val); + if (field.isRepeated()) builder.clearField(field); + addField(builder, field, val); + return new PersistentProtocolBufferMap(meta(), def, builder); } @@ -272,27 +331,26 @@ public IPersistentMap assocEx(Object key, Object val) throws Exception { } public IPersistentCollection cons(Object o) { + DynamicMessage.Builder builder = builder(); + if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry) o; - return assoc(e.getKey(), e.getValue()); + addField(builder, (Keyword) e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector) o; if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); - return assoc(v.nth(0), v.nth(1)); - } - - DynamicMessage.Builder builder = builder(); - - for(ISeq s = RT.seq(o); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); - Keyword key = (Keyword) e.getKey(); - setField(builder, def.fieldDescriptor(key), e.getValue()); + addField(builder, (Keyword) v.nth(0), v.nth(1)); + } else { + for (ISeq s = RT.seq(o); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + addField(builder, (Keyword) e.getKey(), e.getValue()); + } } return new PersistentProtocolBufferMap(meta(), def, builder); } public IPersistentMap without(Object key) throws Exception { - Descriptors.FieldDescriptor field = def.fieldDescriptor((Keyword) key); + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return this; if (field.isRequired()) throw new Exception("Can't remove required field"); From 06170b26c1f678b06fed860d00c86842469950db Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 15 Mar 2010 16:48:40 -0700 Subject: [PATCH 024/525] add tests --- .gitignore | 3 ++- src/test.clj | 14 +++++++++++++ src/test/.gitignore | 1 + src/test/protobuf.clj | 49 +++++++++++++++++++++++++++++++++++++++++++ src/test/test.proto | 22 +++++++++++++++++++ 5 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 src/test.clj create mode 100644 src/test/.gitignore create mode 100644 src/test/protobuf.clj create mode 100644 src/test/test.proto diff --git a/.gitignore b/.gitignore index e666a96..c2a5519 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ lib classes -*.jar \ No newline at end of file +*.jar +*.class \ No newline at end of file diff --git a/src/test.clj b/src/test.clj new file mode 100644 index 0000000..ff560d4 --- /dev/null +++ b/src/test.clj @@ -0,0 +1,14 @@ +(use 'clojure.test) + +(def test-names [:protobuf]) + +(def test-namespaces + (map #(symbol (str "test." (name %))) + test-names)) + +(defn run [] + (println "Loading tests...") + (apply require :reload-all test-namespaces) + (apply run-tests test-namespaces)) + +(run) \ No newline at end of file diff --git a/src/test/.gitignore b/src/test/.gitignore new file mode 100644 index 0000000..289500b --- /dev/null +++ b/src/test/.gitignore @@ -0,0 +1 @@ +Proto.java \ No newline at end of file diff --git a/src/test/protobuf.clj b/src/test/protobuf.clj new file mode 100644 index 0000000..05327ed --- /dev/null +++ b/src/test/protobuf.clj @@ -0,0 +1,49 @@ +(ns test.protobuf + (:use protobuf) + (:use clojure.test)) + +(defprotobuf Foo test.protobuf.Proto Foo) +(defprotobuf Bar test.protobuf.Proto Bar) + +(deftest protobuf-methods + (testing "conj" + (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] + (let [p (conj p {:info "bar"})] + (is (= 5 (:num p))) + (is (= "bar" (:info p))) + (is (= ["little" "yellow"] (:tags p)))) + (let [p (conj p {:tags ["different"]})] + (is (= ["little" "yellow" "different"] (:tags p)))) + (let [p (conj p {:tags ["different"] :info "very"})] + (is (= ["little" "yellow" "different"] (:tags p))) + (is (= "very" (:info p)))) + )) + (testing "assoc" + (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] + (let [p (assoc p :info "baz" :tags ["nuprin"])] + (is (= ["nuprin"] (:tags p))) + (is (= "baz" (:info p)))) + (let [p (assoc p "responses" [:yes :no :maybe :no "yes"])] + (is (= [:yes :no :maybe :no :yes] (:responses p)))) + (let [p (assoc p :tags "aspirin")] + (is (= ["aspirin"] (:tags p)))) + )) + (testing "dissoc" + (let [p (protobuf Foo :num 5 :tags ["fast" "shiny"])] + (let [p (dissoc p :info :tags)] + (is (= [] (:tags p))) + (is (= "" (:info p)))) + )) + (testing "contains?" + (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] + (is (contains? p :num)) + (is (contains? p :tags)) + (is (contains? p {:num 5})) + (is (contains? p {:tags "yellow"})) + (is (not (contains? p {:info "nuprin"}))) + (is (not (contains? p :sdsfsdfd))) + (is (not (contains? p {:num 6}))) + (is (not (contains? p {:tags "big"}))) + )) + + ) diff --git a/src/test/test.proto b/src/test/test.proto new file mode 100644 index 0000000..b105e79 --- /dev/null +++ b/src/test/test.proto @@ -0,0 +1,22 @@ +package test.protobuf; + +option java_outer_classname = "Proto"; + +message Foo { + required uint32 num = 1; + optional string info = 2; + repeated string tags = 3; + repeated Response responses = 4; + + enum Response { + YES = 0; + NO = 1; + MAYBE = 1; + } +} + +message Bar { + repeated string tags = 1; + optional string type = 2; + optional string data = 3; +} \ No newline at end of file From cdb6a27a022e19f11dd9e9d65d1ca60195775362 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 26 Mar 2010 17:35:52 -0700 Subject: [PATCH 025/525] first pass at clojure protobuf type extensions --- build.xml | 19 ++++-- proto/.gitignore | 2 + proto/collections.proto | 10 ++++ .../protobuf/PersistentProtocolBufferMap.java | 59 ++++++++++++++++--- src/test/test.proto | 4 +- 5 files changed, 80 insertions(+), 14 deletions(-) create mode 100644 proto/.gitignore create mode 100644 proto/collections.proto diff --git a/build.xml b/build.xml index ebeb9e6..3511238 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ - + @@ -61,12 +61,19 @@ - - - - + + + - + + + + + + + + + diff --git a/proto/.gitignore b/proto/.gitignore new file mode 100644 index 0000000..9387755 --- /dev/null +++ b/proto/.gitignore @@ -0,0 +1,2 @@ +descriptor.proto +*.java \ No newline at end of file diff --git a/proto/collections.proto b/proto/collections.proto new file mode 100644 index 0000000..49f0cb2 --- /dev/null +++ b/proto/collections.proto @@ -0,0 +1,10 @@ +import "google/protobuf/descriptor.proto"; + +option java_package = "clojure.protobuf"; +option java_outer_classname = "Collections"; + +extend google.protobuf.FieldOptions { + optional bool set = 52001; + optional bool map = 52002; + optional string map_by = 52003; +} \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 1e97ff7..204d1a6 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -20,6 +20,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Descriptors; +import com.google.protobuf.DescriptorProtos; import com.google.protobuf.CodedInputStream; public class PersistentProtocolBufferMap extends APersistentMap { //implements IPersistentSet{ @@ -157,8 +158,8 @@ protected DynamicMessage.Builder builder() { } } - static ConcurrentHashMap enum_to_keyword = new ConcurrentHashMap(); - + static ConcurrentHashMap enum_to_keyword = + new ConcurrentHashMap(); static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { Keyword keyword = enum_to_keyword.get(enum_value); if (keyword == null) { @@ -169,23 +170,67 @@ static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_valu return keyword; } + + static Keyword k_null = Keyword.intern(Symbol.intern("")); + static ConcurrentHashMap map_field_by = + new ConcurrentHashMap(); + static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { + Keyword keyword = map_field_by.get(field); + if (keyword == null) { + String name = field.getOptions().getExtension(Collections.mapBy); + + name = name.toLowerCase().replaceAll("_","-"); + keyword = Keyword.intern(Symbol.intern(name)); + map_field_by.putIfAbsent(field, keyword); + } + return keyword == k_null ? null : keyword; + } + + static Keyword k_key = Keyword.intern(Symbol.intern("key")); + static Keyword k_val = Keyword.intern(Symbol.intern("val")); static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { if (value instanceof List) { List values = (List) value; Iterator iterator = values.iterator(); + List items; + + Keyword map_field_by = mapFieldBy(field); + DescriptorProtos.FieldOptions options = field.getOptions(); + if (map_field_by != null) { + items = new ArrayList(values.size() * 2); - List items = new ArrayList(values.size()); - while (iterator.hasNext()) { - items.add(fromProtoValue(field, iterator.next())); + while (iterator.hasNext()) { + PersistentProtocolBufferMap protobuf = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); + items.add(protobuf.valAt(map_field_by)); + items.add(protobuf); + } + return PersistentHashMap.create(items); + } else if (options.getExtension(Collections.map)) { + items = new ArrayList(values.size() * 2); + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Descriptors.FieldDescriptor key = def.fieldDescriptor(k_key); + Descriptors.FieldDescriptor val = def.fieldDescriptor(k_val); + + while (iterator.hasNext()) { + DynamicMessage message = (DynamicMessage) iterator.next(); + items.add(fromProtoValue(key, message.getField(key))); + items.add(fromProtoValue(val, message.getField(val))); + } + return PersistentHashMap.create(items); + } else { + items = new ArrayList(values.size()); + while (iterator.hasNext()) { + items.add(fromProtoValue(field, iterator.next())); + } + return options.getExtension(Collections.set) ? PersistentHashSet.create(items) : PersistentVector.create(items); } - return PersistentVector.create(items); } else { switch (field.getJavaType()) { case ENUM: Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor) value; return enumToKeyword(e); case MESSAGE: - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); DynamicMessage message = (DynamicMessage) value; // Total hack because getField() doesn't return an empty array for repeated messages. diff --git a/src/test/test.proto b/src/test/test.proto index b105e79..3f1a392 100644 --- a/src/test/test.proto +++ b/src/test/test.proto @@ -1,4 +1,5 @@ package test.protobuf; +import "collections.proto"; option java_outer_classname = "Proto"; @@ -6,7 +7,8 @@ message Foo { required uint32 num = 1; optional string info = 2; repeated string tags = 3; - repeated Response responses = 4; + repeated string tag_set = 4 [(set) = true]; + repeated Response responses = 5; enum Response { YES = 0; From 4fbb4b330644d40199aa85012076a6d7bb9713c6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 29 Mar 2010 18:17:59 -0700 Subject: [PATCH 026/525] collection extensions are working --- README.md | 40 ++++++- build.xml | 2 +- .../protobuf/PersistentProtocolBufferMap.java | 108 +++++++++--------- src/test/protobuf.clj | 62 ++++++---- src/test/test.proto | 27 +++-- 5 files changed, 148 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 2e235ce..ecf6448 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ they are WAY faster to serialize and deserialize than standard Clojure objects. Write a `.proto` file: message Person { - required int32 id = 1; - required string name = 2; + required int32 id = 1; + required string name = 2; optional string email = 3; - repeated string nicknames = 4; + repeated string likes = 4; } Compile the file to Java: @@ -28,6 +28,9 @@ Now you can use the protocol buffer in clojure: (assoc p :name "Bill")) => {:id 4, :name "Bill", :email "bob@example.com"} + (assoc p :likes ["climbing" "running" "jumping"]) + => {:id 4, name "Bob", :email "bob@example.com", :likes ["climbing" "running" "jumping"]} + (def b (protobuf-bytes p)) => # @@ -37,6 +40,37 @@ Now you can use the protocol buffer in clojure: A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. +## Collection extensions + +Clojure-protobuf supports extensions to protocol buffers which provide sets and maps using +repeated fields. To use these, you must import the extension file and include it when compiling. For example: + + import "collections.proto"; + message Photo { + required int32 id = 1; + required string path = 2; + repeated string labels = 3 [(set) = true]; + repeated Attr attrs = 4 [(map) = true]; + repeated Tag tags = 5 [(map_by) = "person_id"]; + + message Attr { + required string key = 1; + optional string val = 2; + } + + message Tag { + required int32 person_id = 1; + optional int32 x_coord = 2; + optional int32 y_coord = 3; + optional int32 width = 4; + optional int32 height = 5; + } + } + +Compile the file to Java: + + protoc --java_out=. -I/proto/ -I. example.proto + ## Installation To download clojure.jar and google's protobuf source automatically and install protoc: diff --git a/build.xml b/build.xml index 3511238..3bed496 100644 --- a/build.xml +++ b/build.xml @@ -91,7 +91,7 @@ - + diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 204d1a6..0677724 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -23,7 +23,7 @@ import com.google.protobuf.DescriptorProtos; import com.google.protobuf.CodedInputStream; -public class PersistentProtocolBufferMap extends APersistentMap { //implements IPersistentSet{ +public class PersistentProtocolBufferMap extends APersistentMap { public static class Def { final Descriptors.Descriptor type; ConcurrentHashMap keyword_to_field; @@ -192,37 +192,46 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object if (value instanceof List) { List values = (List) value; Iterator iterator = values.iterator(); - List items; Keyword map_field_by = mapFieldBy(field); DescriptorProtos.FieldOptions options = field.getOptions(); if (map_field_by != null) { - items = new ArrayList(values.size() * 2); - + ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { - PersistentProtocolBufferMap protobuf = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); - items.add(protobuf.valAt(map_field_by)); - items.add(protobuf); + PersistentProtocolBufferMap val = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); + Object key = val.valAt(map_field_by); + PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(key); + if (existing != null) { + map.assoc(key, existing.cons(val)); + } else { + map.assoc(key, val); + } } - return PersistentHashMap.create(items); + return map.persistent(); } else if (options.getExtension(Collections.map)) { - items = new ArrayList(values.size() * 2); Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - Descriptors.FieldDescriptor key = def.fieldDescriptor(k_key); - Descriptors.FieldDescriptor val = def.fieldDescriptor(k_val); + Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); + Descriptors.FieldDescriptor val_field = def.fieldDescriptor(k_val); + ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { DynamicMessage message = (DynamicMessage) iterator.next(); - items.add(fromProtoValue(key, message.getField(key))); - items.add(fromProtoValue(val, message.getField(val))); + Object key = fromProtoValue(key_field, message.getField(key_field)); + Object val = fromProtoValue(val_field, message.getField(val_field)); + Object existing = map.valAt(key); + if (existing != null && existing instanceof IPersistentCollection) { + map.assoc(key, ((IPersistentCollection) existing).cons(val)); + } else { + map.assoc(key, val); + } } - return PersistentHashMap.create(items); + return map.persistent(); } else { - items = new ArrayList(values.size()); + List list = new ArrayList(values.size()); while (iterator.hasNext()) { - items.add(fromProtoValue(field, iterator.next())); + list.add(fromProtoValue(field, iterator.next())); } - return options.getExtension(Collections.set) ? PersistentHashSet.create(items) : PersistentVector.create(items); + return options.getExtension(Collections.set) ? PersistentHashSet.create(list) : PersistentVector.create(list); } } else { switch (field.getJavaType()) { @@ -285,17 +294,37 @@ protected void addField(DynamicMessage.Builder builder, Descriptors.FieldDescrip if (field == null) return; if (field.isRepeated()) { - if (val instanceof Sequential) { + if (val instanceof Sequential || val instanceof IPersistentSet) { for (ISeq s = RT.seq(val); s != null; s = s.next()) { Object value = toProtoValue(field, s.first()); builder.addRepeatedField(field, value); } } else { - Object value = toProtoValue(field, val); - builder.addRepeatedField(field, value); + Keyword map_field_by = mapFieldBy(field); + if (map_field_by != null) { + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + IPersistentMap map = (IPersistentMap) e.getValue(); + Object value = toProtoValue(field, map.assoc(map_field_by, e.getKey())); + builder.addRepeatedField(field, value); + } + } else if (field.getOptions().getExtension(Collections.map)) { + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; + Object value = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, value); + } + } else { + Object value = toProtoValue(field, val); + builder.addRepeatedField(field, value); + } } } else { Object value = toProtoValue(field, val); + if (value instanceof DynamicMessage) { + value = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) value); + } builder.setField(field, value); } } @@ -305,41 +334,14 @@ public Obj withMeta(IPersistentMap meta) { return new PersistentProtocolBufferMap(meta(), def, message); } - public boolean contains(Object key){ - if (key instanceof IPersistentMap) { - for (ISeq s = RT.seq(key); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); - Object v = e.getValue(); - Object value = valAt(e.getKey()); - - if (value instanceof APersistentVector) { - APersistentVector vec = (APersistentVector) value; - if (!vec.contains(v)) return false; - } else if (value instanceof IPersistentSet) { - IPersistentSet set = (IPersistentSet) value; - if (!set.contains(v)) return false; - } else { - if (value != v) return false; - } - } - return true; - } else { - return false; - } - } - public boolean containsKey(Object key) { - if (key instanceof IPersistentMap) { - return contains(key); + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + if (field == null) { + return false; + } else if (field.isRepeated()) { + return message().getRepeatedFieldCount(field) > 0; } else { - Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) { - return false; - } else if (field.isRepeated()) { - return message().getRepeatedFieldCount(field) > 0; - } else { - return message().hasField(field); - } + return message().hasField(field); } } diff --git a/src/test/protobuf.clj b/src/test/protobuf.clj index 05327ed..3062c3f 100644 --- a/src/test/protobuf.clj +++ b/src/test/protobuf.clj @@ -3,47 +3,59 @@ (:use clojure.test)) (defprotobuf Foo test.protobuf.Proto Foo) -(defprotobuf Bar test.protobuf.Proto Bar) -(deftest protobuf-methods +(deftest protobuf-simple (testing "conj" - (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] - (let [p (conj p {:info "bar"})] - (is (= 5 (:num p))) - (is (= "bar" (:info p))) + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (conj p {:label "bar"})] + (is (= 5 (:id p))) + (is (= "bar" (:label p))) (is (= ["little" "yellow"] (:tags p)))) (let [p (conj p {:tags ["different"]})] (is (= ["little" "yellow" "different"] (:tags p)))) - (let [p (conj p {:tags ["different"] :info "very"})] + (let [p (conj p {:tags ["different"] :label "very"})] (is (= ["little" "yellow" "different"] (:tags p))) - (is (= "very" (:info p)))) + (is (= "very" (:label p)))) )) (testing "assoc" - (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] - (let [p (assoc p :info "baz" :tags ["nuprin"])] + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (assoc p :label "baz" :tags ["nuprin"])] (is (= ["nuprin"] (:tags p))) - (is (= "baz" (:info p)))) + (is (= "baz" (:label p)))) (let [p (assoc p "responses" [:yes :no :maybe :no "yes"])] (is (= [:yes :no :maybe :no :yes] (:responses p)))) (let [p (assoc p :tags "aspirin")] (is (= ["aspirin"] (:tags p)))) )) (testing "dissoc" - (let [p (protobuf Foo :num 5 :tags ["fast" "shiny"])] - (let [p (dissoc p :info :tags)] + (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"])] + (let [p (dissoc p :label :tags)] (is (= [] (:tags p))) - (is (= "" (:info p)))) + (is (= "" (:label p)))) )) - (testing "contains?" - (let [p (protobuf Foo :num 5 :tags ["little" "yellow"])] - (is (contains? p :num)) - (is (contains? p :tags)) - (is (contains? p {:num 5})) - (is (contains? p {:tags "yellow"})) - (is (not (contains? p {:info "nuprin"}))) - (is (not (contains? p :sdsfsdfd))) - (is (not (contains? p {:num 6}))) - (is (not (contains? p {:tags "big"}))) + ) + +(deftest protobuf-extended + (testing "create" + (let [p (protobuf Foo :id 5 :tag-set ["little" "yellow"] :attr-map {"size" "little", "color" "yellow", "style" "different"})] + (is (= #{"little" "yellow"} (:tag-set p))) + (is (associative? (:attr-map p))) + (is (= "different" (get-in p [:attr-map "style"]))) + (is (= "little" (get-in p [:attr-map "size" ]))) + (is (= "yellow" (get-in p [:attr-map "color"]))) + ) + (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five"}, 6 {:label "six"}})] + (let [five ((p :foo-by-id) 5) + six ((p :foo-by-id) 6)] + (is (= 5 (five :id))) + (is (= "five" (five :label))) + (is (= 6 (six :id))) + (is (= "six" (six :label)))) )) + (testing "conj" + (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] + (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["even"]}}})] + (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) + (is (= ["even" "even"] (get-in p [:foo-by-id 6 :tags])))) + ))) - ) diff --git a/src/test/test.proto b/src/test/test.proto index 3f1a392..b526634 100644 --- a/src/test/test.proto +++ b/src/test/test.proto @@ -4,10 +4,10 @@ import "collections.proto"; option java_outer_classname = "Proto"; message Foo { - required uint32 num = 1; - optional string info = 2; - repeated string tags = 3; - repeated string tag_set = 4 [(set) = true]; + required uint32 id = 1; + optional string label = 2; + repeated string tags = 3; + optional Foo parent = 4; repeated Response responses = 5; enum Response { @@ -15,10 +15,19 @@ message Foo { NO = 1; MAYBE = 1; } + + repeated string tag_set = 10 [(set) = true]; + repeated Pair attr_map = 11 [(map) = true]; + repeated Foo foo_by_id = 12 [(map_by) = "id"]; + repeated Group groups = 13 [(map) = true]; +} + +message Pair { + required string key = 1; + required string val = 2; } -message Bar { - repeated string tags = 1; - optional string type = 2; - optional string data = 3; -} \ No newline at end of file +message Group { + required string key = 1; + repeated Foo val = 2; +} From 72a3411e71c1eb8ab76f8ea80d91f944d0356a28 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 30 Mar 2010 15:11:58 -0700 Subject: [PATCH 027/525] add protoclj compiler and ant task for tests --- build.xml | 50 +++++++++++++++++++++++++++----- proto/.gitignore | 2 -- {proto => src}/collections.proto | 2 +- src/protoclj | 16 ++++++++++ src/test/test.proto | 2 +- 5 files changed, 61 insertions(+), 11 deletions(-) delete mode 100644 proto/.gitignore rename {proto => src}/collections.proto (99%) create mode 100755 src/protoclj diff --git a/build.xml b/build.xml index 3bed496..1834117 100644 --- a/build.xml +++ b/build.xml @@ -55,25 +55,43 @@ + + + + + + + + + + + + + + - + + - + - + - - + + - + - + + + + @@ -87,6 +105,7 @@ + @@ -102,4 +121,21 @@ + + + + + + + + + + + + + + + + + diff --git a/proto/.gitignore b/proto/.gitignore deleted file mode 100644 index 9387755..0000000 --- a/proto/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -descriptor.proto -*.java \ No newline at end of file diff --git a/proto/collections.proto b/src/collections.proto similarity index 99% rename from proto/collections.proto rename to src/collections.proto index 49f0cb2..9883606 100644 --- a/proto/collections.proto +++ b/src/collections.proto @@ -7,4 +7,4 @@ extend google.protobuf.FieldOptions { optional bool set = 52001; optional bool map = 52002; optional string map_by = 52003; -} \ No newline at end of file +} diff --git a/src/protoclj b/src/protoclj new file mode 100755 index 0000000..c5d905f --- /dev/null +++ b/src/protoclj @@ -0,0 +1,16 @@ +#!/bin/sh + +if [ -L $0 ]; then + FILE=`readlink $0` +else + FILE=$0 +fi +DIR=`dirname $FILE` +TMP_DIR="/tmp/protoclj-`date +%s`" +SRC_DIR=${2:-'.'} +CLASS_DIR=${2:-$SRC_DIR} + +mkdir -p $TMP_DIR $SRC_DIR $CLASS_DIR +protoc $1 -I$DIR/../include -I. --java_out=$TMP_DIR +cp -Rf $TMP_DIR/ $SRC_DIR +javac -d $CLASS_DIR -cp $DIR/../var/java/clojure-protobuf.jar `find $TMP_DIR -name *.java` \ No newline at end of file diff --git a/src/test/test.proto b/src/test/test.proto index b526634..05a65e0 100644 --- a/src/test/test.proto +++ b/src/test/test.proto @@ -1,5 +1,5 @@ package test.protobuf; -import "collections.proto"; +import "clojure/protobuf/collections.proto"; option java_outer_classname = "Proto"; From 48f24ab02d74c35b13d54aae81fdb3cb1217b505 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 30 Mar 2010 16:56:58 -0700 Subject: [PATCH 028/525] fix install sudo problem --- build.xml | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/build.xml b/build.xml index 1834117..a717c76 100644 --- a/build.xml +++ b/build.xml @@ -53,27 +53,6 @@ - - - - - - - - - - - - - - - - - - - - - @@ -119,6 +98,28 @@ + + + + + + + + + + + + + + + + + + + + + + From d423ab9dd19a78a9c45c95d388ec76604a607d5e Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 30 Mar 2010 17:45:31 -0700 Subject: [PATCH 029/525] fun with ant --- build.xml | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/build.xml b/build.xml index a717c76..7f9a5f2 100644 --- a/build.xml +++ b/build.xml @@ -53,24 +53,31 @@ - + + + + + + + + - - - - - - + + + + - - - - + + + + + + + + + + - - - - @@ -81,14 +88,7 @@ - - - - - - - - + From d94b2b5828391c3984a4e61e725d932363385be2 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 1 Apr 2010 11:19:49 -0700 Subject: [PATCH 030/525] fix build.xml --- build.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml index 7f9a5f2..c2aa14f 100644 --- a/build.xml +++ b/build.xml @@ -62,7 +62,7 @@ - + @@ -72,7 +72,7 @@ - + @@ -123,7 +123,7 @@ - + From e7e80062e10289cb8d0cb3b69aa1224e44a02809 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 2 Apr 2010 19:11:52 -0700 Subject: [PATCH 031/525] gitignore --- src/jvm/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/jvm/.gitignore diff --git a/src/jvm/.gitignore b/src/jvm/.gitignore new file mode 100644 index 0000000..e2cb692 --- /dev/null +++ b/src/jvm/.gitignore @@ -0,0 +1 @@ +Collections.java \ No newline at end of file From dc2aaee05cf8df06f6c624dcbcd43ac11d88b534 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 2 Apr 2010 21:38:21 -0700 Subject: [PATCH 032/525] fix Dependencies passed to FileDescriptor.buildFrom() don't match those listed in the --- .gitignore | 1 + build.xml | 10 +++++----- src/{ => jvm/clojure/protobuf}/collections.proto | 0 3 files changed, 6 insertions(+), 5 deletions(-) rename src/{ => jvm/clojure/protobuf}/collections.proto (100%) diff --git a/.gitignore b/.gitignore index c2a5519..c21b997 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*~ lib classes *.jar diff --git a/build.xml b/build.xml index c2aa14f..7dd67d9 100644 --- a/build.xml +++ b/build.xml @@ -72,11 +72,11 @@ - - + + - - + + @@ -108,7 +108,7 @@ - + diff --git a/src/collections.proto b/src/jvm/clojure/protobuf/collections.proto similarity index 100% rename from src/collections.proto rename to src/jvm/clojure/protobuf/collections.proto From a71f69557b8c9d6ae02789a9102b96e92c22358e Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 6 Apr 2010 13:39:31 -0700 Subject: [PATCH 033/525] allow strings for keys in cons --- build.xml | 3 ++- .../protobuf/PersistentProtocolBufferMap.java | 19 +++++++++---------- src/test/protobuf.clj | 10 ++++++++++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/build.xml b/build.xml index 7dd67d9..c09c8b0 100644 --- a/build.xml +++ b/build.xml @@ -126,7 +126,8 @@ - + + diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 0677724..6a8ca6f 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -68,7 +68,9 @@ public DynamicMessage.Builder newBuilder() { public Descriptors.FieldDescriptor fieldDescriptor(Object key) { if (key == null) return null; - if (key instanceof Keyword) { + if (key instanceof Descriptors.FieldDescriptor) { + return (Descriptors.FieldDescriptor) key; + } else if (key instanceof Keyword) { Keyword keyword = (Keyword) key; Descriptors.FieldDescriptor field = keyword_to_field.get(keyword); if (field == null) { @@ -286,12 +288,9 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v } } - protected void addField(DynamicMessage.Builder builder, Keyword key, Object val) { - addField(builder, def.fieldDescriptor(key), val); - } - - protected void addField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object val) { - if (field == null) return; + protected void addField(DynamicMessage.Builder builder, Object key, Object val) { + if (key == null) return; + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field.isRepeated()) { if (val instanceof Sequential || val instanceof IPersistentSet) { @@ -382,15 +381,15 @@ public IPersistentCollection cons(Object o) { if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry) o; - addField(builder, (Keyword) e.getKey(), e.getValue()); + addField(builder, e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector) o; if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); - addField(builder, (Keyword) v.nth(0), v.nth(1)); + addField(builder, v.nth(0), v.nth(1)); } else { for (ISeq s = RT.seq(o); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); - addField(builder, (Keyword) e.getKey(), e.getValue()); + addField(builder, e.getKey(), e.getValue()); } } return new PersistentProtocolBufferMap(meta(), def, builder); diff --git a/src/test/protobuf.clj b/src/test/protobuf.clj index 3062c3f..7f13c91 100644 --- a/src/test/protobuf.clj +++ b/src/test/protobuf.clj @@ -33,6 +33,16 @@ (is (= [] (:tags p))) (is (= "" (:label p)))) )) + (testing "string keys" + (let [p (protobuf Foo "id" 5 "label" "rad")] + (is (= 5 (p :id))) + (is (= 5 (p "id"))) + (is (= "rad" (p :label))) + (is (= "rad" (p "label"))) + (let [p (conj p {"tags" ["check" "it" "out"]})] + (is (= ["check" "it" "out"] (p :tags))) + (is (= ["check" "it" "out"] (p "tags")))) + )) ) (deftest protobuf-extended From 023d915dc72995ef210cf3dd751f3fed1635d367 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 14 Apr 2010 18:37:06 -0700 Subject: [PATCH 034/525] add protodefaults and protofields methods --- src/clj/protobuf.clj | 10 ++++++++++ .../protobuf/PersistentProtocolBufferMap.java | 12 +++++++++++- src/test/protobuf.clj | 16 +++++++++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index f940df9..ee73d55 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -23,6 +23,16 @@ ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) +(defn protodefault [type key] + (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type)] + (.defaultValue type key))) + +(defn protofields [type] + (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type) + type #^com.google.protobuf.Descriptors$Descriptor (.getMessageType type)] + (map (fn [#^com.google.protobuf.Descriptors$FieldDescriptor field] (keyword (.getName field))) + (.getFields type)))) + (defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data))) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 6a8ca6f..a649857 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -95,9 +95,19 @@ public String getFullName() { public Descriptors.Descriptor getMessageType() { return type; } + + public Object defaultValue(Keyword key) { + Descriptors.FieldDescriptor field = fieldDescriptor(key); + if (field.getType() == Descriptors.FieldDescriptor.Type.MESSAGE) { + if (!field.isRepeated()) return null; + return PersistentProtocolBufferMap.fromProtoValue(field, new ArrayList()); + } else { + return PersistentProtocolBufferMap.fromProtoValue(field, field.getDefaultValue()); + } + } } - final Def def; + final Def def; final DynamicMessage message; DynamicMessage built_message; diff --git a/src/test/protobuf.clj b/src/test/protobuf.clj index 7f13c91..dde33a7 100644 --- a/src/test/protobuf.clj +++ b/src/test/protobuf.clj @@ -43,7 +43,21 @@ (is (= ["check" "it" "out"] (p :tags))) (is (= ["check" "it" "out"] (p "tags")))) )) - ) + (testing "protofields" + (let [fields '(:id :label :tags :parent :responses :tag_set :attr_map :foo_by_id :groups)] + (is (= fields (protofields Foo))) + (is (= fields (protofields test.protobuf.Proto$Foo))))) + (testing "protodefault" + (is (= 0 (protodefault Foo :id))) + (is (= "" (protodefault Foo :label))) + (is (= [] (protodefault Foo :tags))) + (is (= nil (protodefault Foo :parent))) + (is (= [] (protodefault Foo :responses))) + (is (= #{} (protodefault Foo :tag_set))) + (is (= {} (protodefault Foo :foo_by_id))) + (is (= {} (protodefault Foo :groups))) + (is (= {} (protodefault test.protobuf.Proto$Foo :groups))) + )) (deftest protobuf-extended (testing "create" From 724e085ef84240eb51f4e00931ef4a7f3cee11a1 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 15 Apr 2010 16:17:11 -0700 Subject: [PATCH 035/525] add test for append with defaults --- src/test/protobuf.clj | 18 +++++++++++++++++- src/test/test.proto | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/test/protobuf.clj b/src/test/protobuf.clj index dde33a7..38360a0 100644 --- a/src/test/protobuf.clj +++ b/src/test/protobuf.clj @@ -4,6 +4,9 @@ (defprotobuf Foo test.protobuf.Proto Foo) +(defn catbytes [& args] + (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) + (deftest protobuf-simple (testing "conj" (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] @@ -43,12 +46,25 @@ (is (= ["check" "it" "out"] (p :tags))) (is (= ["check" "it" "out"] (p "tags")))) )) + (testing "append" + (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"]) + q (protobuf Foo :id 43 :tags ["savory"]) + r (protobuf Foo :label "bad") + s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) + t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] + (is (= 43 (s :id))) ; make sure an explicit default overwrites on append + (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append + (is (= "rad" (s :label))) + (is (= "bad" (t :label))) + (is (= ["sweet"] (t :tags))) + (is (= ["sweet" "savory"] (s :tags))) + )) (testing "protofields" (let [fields '(:id :label :tags :parent :responses :tag_set :attr_map :foo_by_id :groups)] (is (= fields (protofields Foo))) (is (= fields (protofields test.protobuf.Proto$Foo))))) (testing "protodefault" - (is (= 0 (protodefault Foo :id))) + (is (= 43 (protodefault Foo :id))) (is (= "" (protodefault Foo :label))) (is (= [] (protodefault Foo :tags))) (is (= nil (protodefault Foo :parent))) diff --git a/src/test/test.proto b/src/test/test.proto index 05a65e0..419cc11 100644 --- a/src/test/test.proto +++ b/src/test/test.proto @@ -4,7 +4,7 @@ import "clojure/protobuf/collections.proto"; option java_outer_classname = "Proto"; message Foo { - required uint32 id = 1; + optional uint32 id = 1 [default = 43]; optional string label = 2; repeated string tags = 3; optional Foo parent = 4; From 0c4ada590689f7a5e7b509cf5b7e402f06b66810 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 15 Apr 2010 16:48:26 -0700 Subject: [PATCH 036/525] fix protoclj script --- src/protoclj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protoclj b/src/protoclj index c5d905f..cbde80c 100755 --- a/src/protoclj +++ b/src/protoclj @@ -8,7 +8,7 @@ fi DIR=`dirname $FILE` TMP_DIR="/tmp/protoclj-`date +%s`" SRC_DIR=${2:-'.'} -CLASS_DIR=${2:-$SRC_DIR} +CLASS_DIR=${3:-$SRC_DIR} mkdir -p $TMP_DIR $SRC_DIR $CLASS_DIR protoc $1 -I$DIR/../include -I. --java_out=$TMP_DIR From 5b0714c79cc94178100bd56bf615da05398a0643 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 28 Apr 2010 15:48:02 -0700 Subject: [PATCH 037/525] make cp work on mac and linux --- build.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index c09c8b0..9357555 100644 --- a/build.xml +++ b/build.xml @@ -10,6 +10,11 @@ + + + + + @@ -118,7 +123,7 @@ - + From ec7ac10de6e11fcba42f8a9d1ccc4e324ced984b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 28 Apr 2010 20:28:57 -0700 Subject: [PATCH 038/525] make protoclj work on linux --- src/protoclj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/protoclj b/src/protoclj index cbde80c..fb3c3a9 100755 --- a/src/protoclj +++ b/src/protoclj @@ -12,5 +12,6 @@ CLASS_DIR=${3:-$SRC_DIR} mkdir -p $TMP_DIR $SRC_DIR $CLASS_DIR protoc $1 -I$DIR/../include -I. --java_out=$TMP_DIR -cp -Rf $TMP_DIR/ $SRC_DIR -javac -d $CLASS_DIR -cp $DIR/../var/java/clojure-protobuf.jar `find $TMP_DIR -name *.java` \ No newline at end of file +cp -Rf $TMP_DIR/* $SRC_DIR +javac -d $CLASS_DIR -cp $DIR/../var/java/clojure-protobuf.jar `find $TMP_DIR -name *.java` +rm -rf $TMP_DIR From 4511e6d2602b2c15e52aeda2138d2d645c0fc8e4 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 5 May 2010 13:24:01 -0700 Subject: [PATCH 039/525] update README --- README.md | 24 ++++++++++++++++++++---- examples/example.proto | 23 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ecf6448..210b1dc 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ Compile the file to Java: Now you can use the protocol buffer in clojure: (use 'protobuf) - (defprotobuf person Example Person) + (defprotobuf Person Example$Person) - (def p (protobuf person :id 4 :name "Bob" :email "bob@example.com")) + (def p (protobuf Person :id 4 :name "Bob" :email "bob@example.com")) => {:id 4, :name "Bob", :email "bob@example.com"} (assoc p :name "Bill")) @@ -31,10 +31,10 @@ Now you can use the protocol buffer in clojure: (assoc p :likes ["climbing" "running" "jumping"]) => {:id 4, name "Bob", :email "bob@example.com", :likes ["climbing" "running" "jumping"]} - (def b (protobuf-bytes p)) + (def b (protobuf-dump p)) => # - (protobuf person b) + (protobuf-load Person b) => {:id 4, :name "Bob", :email "bob@example.com"} A protocol buffer map is immutable just like other clojure objects. It is similar to a @@ -71,6 +71,22 @@ Compile the file to Java: protoc --java_out=. -I/proto/ -I. example.proto +Then you can access the maps in clojure: + + (use 'protobuf) + (defprotobuf Photo Example$Photo) + + (def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} + :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} + :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}})) + => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} + + (def b (protobuf-dump p)) + => # + + (protobuf-load Photo b) + => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} + ## Installation To download clojure.jar and google's protobuf source automatically and install protoc: diff --git a/examples/example.proto b/examples/example.proto index 56b85e2..ca639ac 100644 --- a/examples/example.proto +++ b/examples/example.proto @@ -1,3 +1,26 @@ +import "collections.proto"; + +message Photo { + required int32 id = 1; + required string path = 2; + repeated string labels = 3 [(set) = true]; + repeated Attr attrs = 4 [(map) = true]; + repeated Tag tags = 5 [(map_by) = "person_id"]; + + message Attr { + required string key = 1; + optional string val = 2; + } + + message Tag { + required int32 person_id = 1; + optional int32 x_coord = 2; + optional int32 y_coord = 3; + optional int32 width = 4; + optional int32 height = 5; + } +} + message Person { required int32 id = 1; required string name = 2; From 38a5cec472eb507c74274921d33bf300ca7ea744 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 5 May 2010 14:08:16 -0700 Subject: [PATCH 040/525] add protoclj usage info --- README.md | 19 +++++++++++++++---- src/protoclj | 6 ++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 210b1dc..84dbe20 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Write a `.proto` file: Compile the file to Java: - protoc --java_out=. example.proto + protoclj example.proto Now you can use the protocol buffer in clojure: @@ -69,7 +69,7 @@ repeated fields. To use these, you must import the extension file and include it Compile the file to Java: - protoc --java_out=. -I/proto/ -I. example.proto + protoclj example.proto Then you can access the maps in clojure: @@ -87,15 +87,26 @@ Then you can access the maps in clojure: (protobuf-load Photo b) => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} +## protoclj + +When you install clojure-protobuf, a helper script called protoclj is installed that makes +building .proto files with clojure extensions easier. You can use it like this: + + protoclj package/foo.proto [src] [classes] + +Note that if you are using a package in your .proto file, then the file must be in a +directory structure that corresponds to that package and you must call protoclj from the +package root. + ## Installation To download clojure.jar and google's protobuf source automatically and install protoc: ant package - ant install + sudo ant install You can also specify options to configure (like the install prefix): ant package -Dconfigure='--prefix=/opt/local' - ant install + sudo ant install Or you can specify a specific location for clojure.jar or the protobuf source: ant package -Dclojure.jar=$HOME/lib/java/clojure-1.1.0.jar -Dprotobuf=$HOME/Downloads/protobuf-2.3.0 diff --git a/src/protoclj b/src/protoclj index fb3c3a9..92f3c23 100755 --- a/src/protoclj +++ b/src/protoclj @@ -1,10 +1,16 @@ #!/bin/sh +if [ "$1" = "" ]; then + echo "usage: protoclj [protofile] [src_dir] [class_dir]" + exit +fi + if [ -L $0 ]; then FILE=`readlink $0` else FILE=$0 fi + DIR=`dirname $FILE` TMP_DIR="/tmp/protoclj-`date +%s`" SRC_DIR=${2:-'.'} From 95d20b737e85442c63a60bc66960eb8afaf43d9d Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 13 May 2010 18:23:36 -0700 Subject: [PATCH 041/525] lein tasks for compiling protocol buffers --- project.clj | 6 +++ protos/.gitignore | 1 + .../clojure/protobuf/collections.proto | 0 src/leiningen/install_protoc.clj | 43 +++++++++++++++++++ src/leiningen/proto.clj | 40 +++++++++++++++++ 5 files changed, 90 insertions(+) create mode 100644 project.clj create mode 100644 protos/.gitignore rename {src/jvm => protos}/clojure/protobuf/collections.proto (100%) create mode 100644 src/leiningen/install_protoc.clj create mode 100644 src/leiningen/proto.clj diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..b40eec5 --- /dev/null +++ b/project.clj @@ -0,0 +1,6 @@ +(defproject clojure-protobuf "0.0.1-SNAPSHOT" + :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." + :dependencies [[org.clojure/clojure "1.1.0"] + [org.clojure/clojure-contrib "1.1.0"] + [classlojure "0.0.3-SNAPSHOT"]] + :source-path "src/clj") \ No newline at end of file diff --git a/protos/.gitignore b/protos/.gitignore new file mode 100644 index 0000000..2a9bbbd --- /dev/null +++ b/protos/.gitignore @@ -0,0 +1 @@ +*.java \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/collections.proto b/protos/clojure/protobuf/collections.proto similarity index 100% rename from src/jvm/clojure/protobuf/collections.proto rename to protos/clojure/protobuf/collections.proto diff --git a/src/leiningen/install_protoc.clj b/src/leiningen/install_protoc.clj new file mode 100644 index 0000000..d2e1ba4 --- /dev/null +++ b/src/leiningen/install_protoc.clj @@ -0,0 +1,43 @@ +(ns leiningen.install-protoc + (:use clojure.contrib.shell-out + clojure.contrib.str-utils + lancet)) + +(def proto-version "protobuf-2.3.0") +(def build-dir "lib/build/") +(def protobuf-dir (str build-dir proto-version)) + +(defn shell [& args] + (let [return (apply sh :return-map true args)] + (if (= 0 (return :exit)) + (return :out) + (throw (Exception. (str "command failed: " (str-join " " args) "\n" (return :err))))))) + +(defn exists? [file] + (println file) + (.exists (java.io.File. file))) + +(deftarget fetch-source "fetch google protocol buffer source" + (let [url (java.net.URL. (str "http://protobuf.googlecode.com/files/" proto-version ".tar.gz")) + tarfile (str protobuf-dir ".tar.gz") + gzip (doto (org.apache.tools.ant.taskdefs.Untar$UntarCompressionMethod.) (.setValue "gzip"))] + (when-not (exists? protobuf-dir) + (mkdir {:dir build-dir}) + (ant-get {:src url :dest tarfile}) + (untar {:src tarfile :dest build-dir :compression gzip})))) + +(deftarget install "install google protocol buffers" + (when (empty? (sh "which" "protoc")) + (fetch-source) + (with-sh-dir protobuf-dir + (when-not (exists? (str protobuf-dir "/src/protoc")) + (println "building google protobuf compiler...") + (chmod {:file (str protobuf-dir "/configure") :perm "+x"}) + (chmod {:file (str protobuf-dir "/install-sh") :perm "+x"}) + (shell "./configure") + (shell "make")) + (println "installing google protobuf compiler to /usr/local/lib...") + (shell "sudo" "make" "install")))) + +(defn install-protoc [project] + (install)) \ No newline at end of file diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj new file mode 100644 index 0000000..9fcac7d --- /dev/null +++ b/src/leiningen/proto.clj @@ -0,0 +1,40 @@ +(ns leiningen.proto + (:use leiningen.install-protoc + [leiningen.classpath :only [make-path get-classpath]] + [classlojure :only [extract-resource]] + lancet)) + +(def proto-src "protos/src") + +(defn proto-files [dir] + (for [file (rest (file-seq (java.io.File. dir))) + :when (re-find #"\.proto$" (.getName file))] + (.getPath file))) + +(deftarget clojure-protos "extract collections.proto to a temporary directory and return the directory" + ((re-matches #"(.*)/clojure/protobuf/collections.proto" + (extract-resource "protos/clojure/protobuf/collections.proto")) 1)) + +(defn protoc + ([file] (protoc file proto-src)) + ([file out] + (println "compiling" file "to" out) + (mkdir {:dir out}) + (shell "protoc" file + (str "--java_out=" out) + "-I." (str "-I" (clojure-protos))))) + +(defn proto [project] + (install) + (when (= "clojure-protobuf" (project :name)) + (fetch-source) + (let [src (str protobuf-dir "/java/src/main/java")] + (protoc (str protobuf-dir "/src/google/protobuf/descriptor.proto") src) + (javac {:srcdir (make-path src) + :destdir (:compile-path project)}))) + (doseq [file (proto-files "protos")] + (protoc file)) + (javac {:srcdir (make-path proto-src) + :destdir (:compile-path project) + :classpath (apply make-path (get-classpath project))})) + From 3850df93cb16fd77c7feb10bb148d26c8c391fcc Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 12:00:44 -0700 Subject: [PATCH 042/525] this close to ditching lein for good --- project.clj | 29 +++++++++++++++++++++++++++-- src/leiningen/install_protoc.clj | 2 +- src/leiningen/proto.clj | 12 ++++++------ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/project.clj b/project.clj index b40eec5..836daee 100644 --- a/project.clj +++ b/project.clj @@ -2,5 +2,30 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.1.0"] [org.clojure/clojure-contrib "1.1.0"] - [classlojure "0.0.3-SNAPSHOT"]] - :source-path "src/clj") \ No newline at end of file + [classlojure "0.0.4-SNAPSHOT"]] + :source-path "src/clj" + :java-source-path "src/jvm") + +(ns leiningen.compile + (:require lancet) + (:use leiningen.compile + [leiningen.deps :only [deps]] + [leiningen.classpath :only [make-path find-lib-jars get-classpath]]) + (:refer-clojure :exclude [compile])) + +(defn proto [project] + ; needs to happen at runtime because it will fail if "lein deps" hasn't been run + (use ['leiningen.proto :only ['proto]]) + (eval '(proto project))) + +(def compile* compile) +(defn compile [project] + (println (find-lib-jars project)) + (when (empty? (find-lib-jars project)) + (deps project :skip-dev)) + (println (find-lib-jars project)) + (proto project) + (lancet/javac {:srcdir (make-path (:java-source-path project)) + :destdir (:compile-path project) + :classpath (apply make-path (get-classpath project))}) + (compile* project)) diff --git a/src/leiningen/install_protoc.clj b/src/leiningen/install_protoc.clj index d2e1ba4..230b9af 100644 --- a/src/leiningen/install_protoc.clj +++ b/src/leiningen/install_protoc.clj @@ -36,7 +36,7 @@ (chmod {:file (str protobuf-dir "/install-sh") :perm "+x"}) (shell "./configure") (shell "make")) - (println "installing google protobuf compiler to /usr/local/lib...") + (println "installing google protobuf compiler...") (shell "sudo" "make" "install")))) (defn install-protoc [project] diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj index 9fcac7d..e1d1ecb 100644 --- a/src/leiningen/proto.clj +++ b/src/leiningen/proto.clj @@ -4,7 +4,7 @@ [classlojure :only [extract-resource]] lancet)) -(def proto-src "protos/src") +(def src-dir "classes") ; so lein clean will remove them (defn proto-files [dir] (for [file (rest (file-seq (java.io.File. dir))) @@ -16,7 +16,7 @@ (extract-resource "protos/clojure/protobuf/collections.proto")) 1)) (defn protoc - ([file] (protoc file proto-src)) + ([file] (protoc file src-dir)) ([file out] (println "compiling" file "to" out) (mkdir {:dir out}) @@ -28,13 +28,13 @@ (install) (when (= "clojure-protobuf" (project :name)) (fetch-source) - (let [src (str protobuf-dir "/java/src/main/java")] - (protoc (str protobuf-dir "/src/google/protobuf/descriptor.proto") src) - (javac {:srcdir (make-path src) + (let [src-dir (str protobuf-dir "/java/src/main/java")] +; (protoc (str protobuf-dir "/src/google/protobuf/descriptor.proto") src-dir) + (javac {:srcdir (make-path src-dir) :destdir (:compile-path project)}))) (doseq [file (proto-files "protos")] (protoc file)) - (javac {:srcdir (make-path proto-src) + (javac {:srcdir (make-path src-dir) :destdir (:compile-path project) :classpath (apply make-path (get-classpath project))})) From bd8d98f15272a77279470358775fc53576c271a5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 17:58:11 -0700 Subject: [PATCH 043/525] ove lein tasks out into separate project --- project.clj | 32 ++++++-------- protos/.gitignore | 3 +- .../protobuf/PersistentProtocolBufferMap.java | 15 ++++--- src/leiningen/install_protoc.clj | 43 ------------------- src/leiningen/proto.clj | 40 ----------------- src/protoclj | 23 ---------- {src/test => test}/.gitignore | 0 {src/test => test}/protobuf.clj | 0 {src/test => test}/test.proto | 0 9 files changed, 26 insertions(+), 130 deletions(-) delete mode 100644 src/leiningen/install_protoc.clj delete mode 100644 src/leiningen/proto.clj delete mode 100755 src/protoclj rename {src/test => test}/.gitignore (100%) rename {src/test => test}/protobuf.clj (100%) rename {src/test => test}/test.proto (100%) diff --git a/project.clj b/project.clj index 836daee..65a1888 100644 --- a/project.clj +++ b/project.clj @@ -1,31 +1,27 @@ (defproject clojure-protobuf "0.0.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[org.clojure/clojure "1.1.0"] - [org.clojure/clojure-contrib "1.1.0"] - [classlojure "0.0.4-SNAPSHOT"]] + :dependencies [[clojure "1.2.0-master-SNAPSHOT"] + [clojure-contrib "1.2.0-SNAPSHOT"]] + :dev-dependencies [[lein-protobuf "0.0.2-SNAPSHOT"]] :source-path "src/clj" :java-source-path "src/jvm") (ns leiningen.compile (:require lancet) - (:use leiningen.compile - [leiningen.deps :only [deps]] - [leiningen.classpath :only [make-path find-lib-jars get-classpath]]) + (:use [leiningen.classpath :only [make-path get-classpath]]) (:refer-clojure :exclude [compile])) -(defn proto [project] - ; needs to happen at runtime because it will fail if "lein deps" hasn't been run - (use ['leiningen.proto :only ['proto]]) - (eval '(proto project))) +(defn compile-protobuf [project] + (try + (require 'leiningen.proto) + ((ns-resolve 'leiningen.proto 'proto) project) + (catch java.io.FileNotFoundException e + (println "you must run 'lein deps' before compile") + (System/exit 1)))) -(def compile* compile) (defn compile [project] - (println (find-lib-jars project)) - (when (empty? (find-lib-jars project)) - (deps project :skip-dev)) - (println (find-lib-jars project)) - (proto project) + (println (get-classpath project)) + (compile-protobuf project) (lancet/javac {:srcdir (make-path (:java-source-path project)) :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))}) - (compile* project)) + :classpath (apply make-path (get-classpath project))})) \ No newline at end of file diff --git a/protos/.gitignore b/protos/.gitignore index 2a9bbbd..dd7c53b 100644 --- a/protos/.gitignore +++ b/protos/.gitignore @@ -1 +1,2 @@ -*.java \ No newline at end of file +*.java +google/protobuf/descriptor.proto \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index a649857..cc210c1 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -109,6 +109,7 @@ public Object defaultValue(Keyword key) { final Def def; final DynamicMessage message; + final IPersistentMap _meta; DynamicMessage built_message; @@ -129,19 +130,19 @@ static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyv } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { - super(meta); + this._meta = meta; this.def = def; this.message = null; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message) { - super(meta); + this._meta = meta; this.def = def; this.message = message; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage.Builder builder) { - super(meta); + this._meta = meta; this.def = def; this.message = builder.build(); } @@ -338,11 +339,15 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) } } - public Obj withMeta(IPersistentMap meta) { + public PersistentProtocolBufferMap withMeta(IPersistentMap meta) { if (meta == meta()) return this; return new PersistentProtocolBufferMap(meta(), def, message); } + public IPersistentMap meta(){ + return _meta; + } + public boolean containsKey(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) { @@ -458,7 +463,7 @@ public Obj withMeta(IPersistentMap meta) { if(meta != meta()) return new Seq(meta, map, fields, i); return this; } - + public Object first() { if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; diff --git a/src/leiningen/install_protoc.clj b/src/leiningen/install_protoc.clj deleted file mode 100644 index 230b9af..0000000 --- a/src/leiningen/install_protoc.clj +++ /dev/null @@ -1,43 +0,0 @@ -(ns leiningen.install-protoc - (:use clojure.contrib.shell-out - clojure.contrib.str-utils - lancet)) - -(def proto-version "protobuf-2.3.0") -(def build-dir "lib/build/") -(def protobuf-dir (str build-dir proto-version)) - -(defn shell [& args] - (let [return (apply sh :return-map true args)] - (if (= 0 (return :exit)) - (return :out) - (throw (Exception. (str "command failed: " (str-join " " args) "\n" (return :err))))))) - -(defn exists? [file] - (println file) - (.exists (java.io.File. file))) - -(deftarget fetch-source "fetch google protocol buffer source" - (let [url (java.net.URL. (str "http://protobuf.googlecode.com/files/" proto-version ".tar.gz")) - tarfile (str protobuf-dir ".tar.gz") - gzip (doto (org.apache.tools.ant.taskdefs.Untar$UntarCompressionMethod.) (.setValue "gzip"))] - (when-not (exists? protobuf-dir) - (mkdir {:dir build-dir}) - (ant-get {:src url :dest tarfile}) - (untar {:src tarfile :dest build-dir :compression gzip})))) - -(deftarget install "install google protocol buffers" - (when (empty? (sh "which" "protoc")) - (fetch-source) - (with-sh-dir protobuf-dir - (when-not (exists? (str protobuf-dir "/src/protoc")) - (println "building google protobuf compiler...") - (chmod {:file (str protobuf-dir "/configure") :perm "+x"}) - (chmod {:file (str protobuf-dir "/install-sh") :perm "+x"}) - (shell "./configure") - (shell "make")) - (println "installing google protobuf compiler...") - (shell "sudo" "make" "install")))) - -(defn install-protoc [project] - (install)) \ No newline at end of file diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj deleted file mode 100644 index e1d1ecb..0000000 --- a/src/leiningen/proto.clj +++ /dev/null @@ -1,40 +0,0 @@ -(ns leiningen.proto - (:use leiningen.install-protoc - [leiningen.classpath :only [make-path get-classpath]] - [classlojure :only [extract-resource]] - lancet)) - -(def src-dir "classes") ; so lein clean will remove them - -(defn proto-files [dir] - (for [file (rest (file-seq (java.io.File. dir))) - :when (re-find #"\.proto$" (.getName file))] - (.getPath file))) - -(deftarget clojure-protos "extract collections.proto to a temporary directory and return the directory" - ((re-matches #"(.*)/clojure/protobuf/collections.proto" - (extract-resource "protos/clojure/protobuf/collections.proto")) 1)) - -(defn protoc - ([file] (protoc file src-dir)) - ([file out] - (println "compiling" file "to" out) - (mkdir {:dir out}) - (shell "protoc" file - (str "--java_out=" out) - "-I." (str "-I" (clojure-protos))))) - -(defn proto [project] - (install) - (when (= "clojure-protobuf" (project :name)) - (fetch-source) - (let [src-dir (str protobuf-dir "/java/src/main/java")] -; (protoc (str protobuf-dir "/src/google/protobuf/descriptor.proto") src-dir) - (javac {:srcdir (make-path src-dir) - :destdir (:compile-path project)}))) - (doseq [file (proto-files "protos")] - (protoc file)) - (javac {:srcdir (make-path src-dir) - :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))})) - diff --git a/src/protoclj b/src/protoclj deleted file mode 100755 index 92f3c23..0000000 --- a/src/protoclj +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -if [ "$1" = "" ]; then - echo "usage: protoclj [protofile] [src_dir] [class_dir]" - exit -fi - -if [ -L $0 ]; then - FILE=`readlink $0` -else - FILE=$0 -fi - -DIR=`dirname $FILE` -TMP_DIR="/tmp/protoclj-`date +%s`" -SRC_DIR=${2:-'.'} -CLASS_DIR=${3:-$SRC_DIR} - -mkdir -p $TMP_DIR $SRC_DIR $CLASS_DIR -protoc $1 -I$DIR/../include -I. --java_out=$TMP_DIR -cp -Rf $TMP_DIR/* $SRC_DIR -javac -d $CLASS_DIR -cp $DIR/../var/java/clojure-protobuf.jar `find $TMP_DIR -name *.java` -rm -rf $TMP_DIR diff --git a/src/test/.gitignore b/test/.gitignore similarity index 100% rename from src/test/.gitignore rename to test/.gitignore diff --git a/src/test/protobuf.clj b/test/protobuf.clj similarity index 100% rename from src/test/protobuf.clj rename to test/protobuf.clj diff --git a/src/test/test.proto b/test/test.proto similarity index 100% rename from src/test/test.proto rename to test/test.proto From 0a891bb95542050d4526c24d0e5c96e1b450b327 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 18:02:18 -0700 Subject: [PATCH 044/525] remove build.xml --- build.xml | 148 ------------------------------------------------------ 1 file changed, 148 deletions(-) delete mode 100644 build.xml diff --git a/build.xml b/build.xml deleted file mode 100644 index 9357555..0000000 --- a/build.xml +++ /dev/null @@ -1,148 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 0ca894cdb0f664cb10c8107dd0c15b733a08a58f Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 18:16:58 -0700 Subject: [PATCH 045/525] update readme --- README.md | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 84dbe20..a2bb810 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ Write a `.proto` file: repeated string likes = 4; } -Compile the file to Java: +If you put it in the protos directory of your project, you can compile it with lein: - protoclj example.proto + lein proto example.proto Now you can use the protocol buffer in clojure: @@ -67,9 +67,9 @@ repeated fields. To use these, you must import the extension file and include it } } -Compile the file to Java: +Compile the file: - protoclj example.proto + lein proto example.proto Then you can access the maps in clojure: @@ -87,28 +87,15 @@ Then you can access the maps in clojure: (protobuf-load Photo b) => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} -## protoclj - -When you install clojure-protobuf, a helper script called protoclj is installed that makes -building .proto files with clojure extensions easier. You can use it like this: - - protoclj package/foo.proto [src] [classes] - -Note that if you are using a package in your .proto file, then the file must be in a -directory structure that corresponds to that package and you must call protoclj from the -package root. - ## Installation -To download clojure.jar and google's protobuf source automatically and install protoc: - ant package - sudo ant install +[Leiningen](http://github.com/technomancy/leiningen): + + :dependencies [[compojure "0.4.0-SNAPSHOT"]] -You can also specify options to configure (like the install prefix): - ant package -Dconfigure='--prefix=/opt/local' - sudo ant install +Build from source: -Or you can specify a specific location for clojure.jar or the protobuf source: - ant package -Dclojure.jar=$HOME/lib/java/clojure-1.1.0.jar -Dprotobuf=$HOME/Downloads/protobuf-2.3.0 + lein deps + lein jar -This code has been tested with clojure version 1.1.0 and protobuf version 2.3.0. +This code works with Clojure 1.2. Here's the [1.1 branch](http://github.com/ninjudd/clojure-protobuf/tree/clojure-1.1). \ No newline at end of file From fff0f296e3db7fea47b76cf0607cfbbd66591739 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 18:18:39 -0700 Subject: [PATCH 046/525] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a2bb810..fb44024 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ Then you can access the maps in clojure: ## Installation -[Leiningen](http://github.com/technomancy/leiningen): +In your [Leiningen](http://github.com/technomancy/leiningen) project.clj: - :dependencies [[compojure "0.4.0-SNAPSHOT"]] + :dependencies [[compojure "0.0.1-SNAPSHOT"]] Build from source: From e3d1e5dbe16f9b4e3e5e8af2465baf4ee40c8d21 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 18:19:19 -0700 Subject: [PATCH 047/525] copy and paste --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb44024..358348b 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Then you can access the maps in clojure: In your [Leiningen](http://github.com/technomancy/leiningen) project.clj: - :dependencies [[compojure "0.0.1-SNAPSHOT"]] + :dependencies [[clojure-protobuf "0.0.1-SNAPSHOT"]] Build from source: From 7769153a0de90cfaa5468eb451b06938d36a8184 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 14 May 2010 18:31:19 -0700 Subject: [PATCH 048/525] update versions --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 65a1888..7d4f983 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject clojure-protobuf "0.0.1-SNAPSHOT" +(defproject clojure-protobuf "0.0.4-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] - :dev-dependencies [[lein-protobuf "0.0.2-SNAPSHOT"]] + :dev-dependencies [[lein-protobuf "0.0.4-SNAPSHOT"]] :source-path "src/clj" :java-source-path "src/jvm") From 8c2ee0e63263f16a2ef53ba565b45343cc2b4bc2 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 17 May 2010 15:15:51 -0700 Subject: [PATCH 049/525] add lein tasks back --- .gitignore | 1 + project.clj | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index c21b997..a269431 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *~ lib classes +pom.xml *.jar *.class \ No newline at end of file diff --git a/project.clj b/project.clj index 7d4f983..508c410 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject clojure-protobuf "0.0.4-SNAPSHOT" +(defproject clojure-protobuf "0.1.0-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] - [clojure-contrib "1.2.0-SNAPSHOT"]] - :dev-dependencies [[lein-protobuf "0.0.4-SNAPSHOT"]] + [clojure-contrib "1.2.0-SNAPSHOT"] + [classlojure "0.0.4-SNAPSHOT"]] :source-path "src/clj" :java-source-path "src/jvm") @@ -20,7 +20,6 @@ (System/exit 1)))) (defn compile [project] - (println (get-classpath project)) (compile-protobuf project) (lancet/javac {:srcdir (make-path (:java-source-path project)) :destdir (:compile-path project) From 3eee9ccc079e760ae924fae4c795d4ba9afaf524 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 18 May 2010 11:48:02 -0700 Subject: [PATCH 050/525] use with-ns to override compile so that jar will call my compile. add missing leiningen tasks --- README.md | 2 +- project.clj | 32 +++++++--------- protos/.gitignore | 1 - src/leiningen/install_protoc.clj | 42 +++++++++++++++++++++ src/leiningen/proto.clj | 64 ++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 21 deletions(-) create mode 100644 src/leiningen/install_protoc.clj create mode 100644 src/leiningen/proto.clj diff --git a/README.md b/README.md index 358348b..42f908f 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ Then you can access the maps in clojure: In your [Leiningen](http://github.com/technomancy/leiningen) project.clj: - :dependencies [[clojure-protobuf "0.0.1-SNAPSHOT"]] + :dependencies [[clojure-protobuf "0.1.0-SNAPSHOT"]] Build from source: diff --git a/project.clj b/project.clj index 508c410..74e0be4 100644 --- a/project.clj +++ b/project.clj @@ -3,24 +3,18 @@ :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"] [classlojure "0.0.4-SNAPSHOT"]] - :source-path "src/clj" - :java-source-path "src/jvm") + :source-path "src/clj" + :java-source-path "src/jvm" + :resources-path "protos") -(ns leiningen.compile - (:require lancet) - (:use [leiningen.classpath :only [make-path get-classpath]]) - (:refer-clojure :exclude [compile])) +(use 'clojure.contrib.with-ns) +(require 'leiningen.compile) +(with-ns 'leiningen.compile + (use '[leiningen.proto :only [proto]]) -(defn compile-protobuf [project] - (try - (require 'leiningen.proto) - ((ns-resolve 'leiningen.proto 'proto) project) - (catch java.io.FileNotFoundException e - (println "you must run 'lein deps' before compile") - (System/exit 1)))) - -(defn compile [project] - (compile-protobuf project) - (lancet/javac {:srcdir (make-path (:java-source-path project)) - :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))})) \ No newline at end of file + (defn compile [project] + (deps project :skip-dev) + (proto project) + (lancet/javac {:srcdir (make-path (:java-source-path project)) + :destdir (:compile-path project) + :classpath (apply make-path (get-classpath project))}))) \ No newline at end of file diff --git a/protos/.gitignore b/protos/.gitignore index dd7c53b..576998b 100644 --- a/protos/.gitignore +++ b/protos/.gitignore @@ -1,2 +1 @@ -*.java google/protobuf/descriptor.proto \ No newline at end of file diff --git a/src/leiningen/install_protoc.clj b/src/leiningen/install_protoc.clj new file mode 100644 index 0000000..1d267f1 --- /dev/null +++ b/src/leiningen/install_protoc.clj @@ -0,0 +1,42 @@ +(ns leiningen.install-protoc + (:use clojure.contrib.shell-out + clojure.contrib.str-utils + lancet)) + +(def proto-version "protobuf-2.3.0") +(def build-dir "lib/build/") +(def protobuf-dir (str build-dir proto-version)) + +(defn shell [& args] + (let [return (apply sh :return-map true args)] + (if (= 0 (return :exit)) + (return :out) + (throw (Exception. (str "command failed: " (str-join " " args) "\n" (return :err))))))) + +(defn exists? [file] + (.exists (java.io.File. file))) + +(deftarget fetch-source "fetch google protocol buffer source" + (let [url (java.net.URL. (str "http://protobuf.googlecode.com/files/" proto-version ".tar.gz")) + tarfile (str protobuf-dir ".tar.gz") + gzip (doto (org.apache.tools.ant.taskdefs.Untar$UntarCompressionMethod.) (.setValue "gzip"))] + (when-not (exists? protobuf-dir) + (mkdir {:dir build-dir}) + (ant-get {:src url :dest tarfile}) + (untar {:src tarfile :dest build-dir :compression gzip})))) + +(deftarget install "install google protocol buffers" + (when (empty? (sh "which" "protoc")) + (fetch-source) + (with-sh-dir protobuf-dir + (when-not (exists? (str protobuf-dir "/src/protoc")) + (println "building google protobuf compiler...") + (chmod {:file (str protobuf-dir "/configure") :perm "+x"}) + (chmod {:file (str protobuf-dir "/install-sh") :perm "+x"}) + (shell "./configure") + (shell "make")) + (println "installing google protobuf compiler...") + (shell "sudo" "make" "install")))) + +(defn install-protoc [project] + (install)) \ No newline at end of file diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj new file mode 100644 index 0000000..16c2aa5 --- /dev/null +++ b/src/leiningen/proto.clj @@ -0,0 +1,64 @@ +(ns leiningen.proto + (:use leiningen.install-protoc + [leiningen.classpath :only [make-path get-classpath]] + [classlojure :only [extract-resource]] + [clojure.contrib.io :only [file read-lines]] + lancet)) + +(defn proto-files [dir] + (for [file (rest (file-seq (file dir))) :when (re-find #"\.proto$" (.getName file))] + (.substring (.getPath file) (inc (count dir))))) + +(defn- match [p s] + (when s + (let [p (if (string? p) (re-pattern p) p)] + (second (re-matches p s))))) + +(defn extract-proto + "extract proto-file to a temporary directory and return the directory" + [proto-file] + (match (str "(.*)/" proto-file) + (extract-resource proto-file))) + +(defn proto-dependencies + "look for lines starting with import in proto-file" + [proto-file] + (for [line (read-lines (file "protos" proto-file)) :when (.startsWith line "import")] + (match #".*\"(.*)\".*" line))) + +(defn protoc [proto-file src-dir] + (println "compiling" proto-file "to" src-dir) + (mkdir {:dir src-dir}) + (let [src-dir (str "../" src-dir)] + (apply shell :dir "protos" + "protoc" proto-file (str "--java_out=" src-dir) "-I." + (for [dir (map extract-proto (proto-dependencies proto-file)) :when dir] + (str "-I" dir))))) + +(declare proto) + +(defn build-protobuf [project] + (fetch-source) + (let [src-dir (str protobuf-dir "/java/src/main/java") + descriptor "google/protobuf/descriptor.proto" + todir "protos/google/protobuf"] + (mkdir {:dir todir}) + (copy {:file (str protobuf-dir "/src/" descriptor) :todir todir}) + (protoc descriptor src-dir) + (javac {:srcdir (make-path src-dir) + :destdir (:compile-path project)}) + (proto project "clojure/protobuf/collections.proto"))) + +(defn proto + ([project] + (if (= "clojure-protobuf" (:name project)) + (build-protobuf project) + (apply proto project (proto-files "protos")))) + ([project & files] + (install) + (let [src-dir (or (:proto-source-dir project) "lib/src")] + (doseq [file files] + (protoc file src-dir)) + (javac {:srcdir (make-path src-dir) + :destdir (:compile-path project) + :classpath (apply make-path (get-classpath project))})))) From 3798cd949fa0561a1ad3ade6784de98c8efe89f2 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 18 May 2010 12:46:35 -0700 Subject: [PATCH 051/525] rename proto dir protos->proto --- project.clj | 2 +- {protos => proto}/.gitignore | 0 {protos => proto}/clojure/protobuf/collections.proto | 0 src/leiningen/proto.clj | 8 ++++---- 4 files changed, 5 insertions(+), 5 deletions(-) rename {protos => proto}/.gitignore (100%) rename {protos => proto}/clojure/protobuf/collections.proto (100%) diff --git a/project.clj b/project.clj index 74e0be4..1611c83 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ [classlojure "0.0.4-SNAPSHOT"]] :source-path "src/clj" :java-source-path "src/jvm" - :resources-path "protos") + :resources-path "proto") (use 'clojure.contrib.with-ns) (require 'leiningen.compile) diff --git a/protos/.gitignore b/proto/.gitignore similarity index 100% rename from protos/.gitignore rename to proto/.gitignore diff --git a/protos/clojure/protobuf/collections.proto b/proto/clojure/protobuf/collections.proto similarity index 100% rename from protos/clojure/protobuf/collections.proto rename to proto/clojure/protobuf/collections.proto diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj index 16c2aa5..bd38119 100644 --- a/src/leiningen/proto.clj +++ b/src/leiningen/proto.clj @@ -23,14 +23,14 @@ (defn proto-dependencies "look for lines starting with import in proto-file" [proto-file] - (for [line (read-lines (file "protos" proto-file)) :when (.startsWith line "import")] + (for [line (read-lines (file "proto" proto-file)) :when (.startsWith line "import")] (match #".*\"(.*)\".*" line))) (defn protoc [proto-file src-dir] (println "compiling" proto-file "to" src-dir) (mkdir {:dir src-dir}) (let [src-dir (str "../" src-dir)] - (apply shell :dir "protos" + (apply shell :dir "proto" "protoc" proto-file (str "--java_out=" src-dir) "-I." (for [dir (map extract-proto (proto-dependencies proto-file)) :when dir] (str "-I" dir))))) @@ -41,7 +41,7 @@ (fetch-source) (let [src-dir (str protobuf-dir "/java/src/main/java") descriptor "google/protobuf/descriptor.proto" - todir "protos/google/protobuf"] + todir "proto/google/protobuf"] (mkdir {:dir todir}) (copy {:file (str protobuf-dir "/src/" descriptor) :todir todir}) (protoc descriptor src-dir) @@ -53,7 +53,7 @@ ([project] (if (= "clojure-protobuf" (:name project)) (build-protobuf project) - (apply proto project (proto-files "protos")))) + (apply proto project (proto-files "proto")))) ([project & files] (install) (let [src-dir (or (:proto-source-dir project) "lib/src")] From f9a6693c7d5c8901fc26d0c92f56ca9c37cbcc2a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 18 May 2010 16:40:57 -0700 Subject: [PATCH 052/525] recursively extract all proto dependencies from classpath --- project.clj | 15 +++-- src/{ => clj}/leiningen/install_protoc.clj | 0 src/clj/leiningen/proto.clj | 75 ++++++++++++++++++++++ src/leiningen | 1 + src/leiningen/proto.clj | 64 ------------------ src/test.clj | 14 ---- 6 files changed, 87 insertions(+), 82 deletions(-) rename src/{ => clj}/leiningen/install_protoc.clj (100%) create mode 100644 src/clj/leiningen/proto.clj create mode 120000 src/leiningen delete mode 100644 src/leiningen/proto.clj delete mode 100644 src/test.clj diff --git a/project.clj b/project.clj index 1611c83..3bf68d6 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"] - [classlojure "0.0.4-SNAPSHOT"]] + [classlojure "0.1.0-SNAPSHOT"]] :source-path "src/clj" :java-source-path "src/jvm" :resources-path "proto") @@ -10,11 +10,18 @@ (use 'clojure.contrib.with-ns) (require 'leiningen.compile) (with-ns 'leiningen.compile - (use '[leiningen.proto :only [proto]]) + + (defn proto [project] + (try (require 'classlojure) + (require 'leiningen.proto) + ((ns-resolve 'leiningen.proto 'proto) project) + (catch java.io.FileNotFoundException e + (println "you must run 'lein deps' first") + (System/exit 1)))) (defn compile [project] - (deps project :skip-dev) (proto project) (lancet/javac {:srcdir (make-path (:java-source-path project)) :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))}))) \ No newline at end of file + :classpath (apply make-path (get-classpath project))})) +) \ No newline at end of file diff --git a/src/leiningen/install_protoc.clj b/src/clj/leiningen/install_protoc.clj similarity index 100% rename from src/leiningen/install_protoc.clj rename to src/clj/leiningen/install_protoc.clj diff --git a/src/clj/leiningen/proto.clj b/src/clj/leiningen/proto.clj new file mode 100644 index 0000000..a53be76 --- /dev/null +++ b/src/clj/leiningen/proto.clj @@ -0,0 +1,75 @@ +(ns leiningen.proto + (:use leiningen.install-protoc + [leiningen.classpath :only [make-path get-classpath]] + [classlojure :only [extract-resource]] + [clojure.contrib.io :only [file read-lines]] + lancet)) + +(def proto-dir "proto") +(def proto-extract-dir "lib/proto") +(def proto-src-dir "lib/proto_src") + +(defn- match [p s] + (when s + (let [p (if (string? p) (re-pattern p) p)] + (second (re-matches p s))))) + +(defn- proto-exists? [proto] + (or (.exists (file proto-dir proto)) + (.exists (file proto-extract-dir proto)))) + +(defn- proto-dependencies + "look for lines starting with import in proto-file" + [proto-file] + (for [line (read-lines proto-file) :when (.startsWith line "import")] + (match #".*\"(.*)\".*" line))) + +(defn extract-dependencies + "extract all files proto is dependent on" + [proto] + (loop [files (vec (proto-dependencies (file proto-dir proto)))] + (when-not (empty? files) + (let [proto (peek files) + files (pop files)] + (if (proto-exists? proto) + (recur files) + (let [proto-file (extract-resource proto proto-extract-dir)] + (recur (into files (proto-dependencies proto-file))))))))) + +(defn protoc [proto src-dir] + (println "compiling" proto "to" src-dir) + (extract-dependencies proto) + (mkdir {:dir src-dir}) + (shell :dir proto-dir + "protoc" proto (str "--java_out=../" src-dir) "-I." (str "-I../" proto-extract-dir))) + +(declare proto) + +(defn build-protobuf [project] + (fetch-source) + (let [src-dir (str protobuf-dir "/java/src/main/java") + descriptor "google/protobuf/descriptor.proto" + todir (str proto-dir "/google/protobuf")] + (mkdir {:dir todir}) + (copy {:file (str protobuf-dir "/src/" descriptor) :todir todir}) + (protoc descriptor src-dir) + (javac {:srcdir (make-path src-dir) + :destdir (:compile-path project)}) + (proto project "clojure/protobuf/collections.proto"))) + +(defn- proto-files [dir] + (for [file (rest (file-seq (file dir))) :when (re-find #"\.proto$" (.getName file))] + (.substring (.getPath file) (inc (count dir))))) + +(defn proto + ([project] + (if (= "clojure-protobuf" (:name project)) + (build-protobuf project) + (apply proto project (proto-files proto-dir)))) + ([project & files] + (install) + (doseq [file files] + (protoc file proto-src-dir)) + (javac {:srcdir (make-path proto-src-dir) + :destdir (:compile-path project) + :classpath (apply make-path (get-classpath project))}))) diff --git a/src/leiningen b/src/leiningen new file mode 120000 index 0000000..369c96c --- /dev/null +++ b/src/leiningen @@ -0,0 +1 @@ +clj/leiningen \ No newline at end of file diff --git a/src/leiningen/proto.clj b/src/leiningen/proto.clj deleted file mode 100644 index bd38119..0000000 --- a/src/leiningen/proto.clj +++ /dev/null @@ -1,64 +0,0 @@ -(ns leiningen.proto - (:use leiningen.install-protoc - [leiningen.classpath :only [make-path get-classpath]] - [classlojure :only [extract-resource]] - [clojure.contrib.io :only [file read-lines]] - lancet)) - -(defn proto-files [dir] - (for [file (rest (file-seq (file dir))) :when (re-find #"\.proto$" (.getName file))] - (.substring (.getPath file) (inc (count dir))))) - -(defn- match [p s] - (when s - (let [p (if (string? p) (re-pattern p) p)] - (second (re-matches p s))))) - -(defn extract-proto - "extract proto-file to a temporary directory and return the directory" - [proto-file] - (match (str "(.*)/" proto-file) - (extract-resource proto-file))) - -(defn proto-dependencies - "look for lines starting with import in proto-file" - [proto-file] - (for [line (read-lines (file "proto" proto-file)) :when (.startsWith line "import")] - (match #".*\"(.*)\".*" line))) - -(defn protoc [proto-file src-dir] - (println "compiling" proto-file "to" src-dir) - (mkdir {:dir src-dir}) - (let [src-dir (str "../" src-dir)] - (apply shell :dir "proto" - "protoc" proto-file (str "--java_out=" src-dir) "-I." - (for [dir (map extract-proto (proto-dependencies proto-file)) :when dir] - (str "-I" dir))))) - -(declare proto) - -(defn build-protobuf [project] - (fetch-source) - (let [src-dir (str protobuf-dir "/java/src/main/java") - descriptor "google/protobuf/descriptor.proto" - todir "proto/google/protobuf"] - (mkdir {:dir todir}) - (copy {:file (str protobuf-dir "/src/" descriptor) :todir todir}) - (protoc descriptor src-dir) - (javac {:srcdir (make-path src-dir) - :destdir (:compile-path project)}) - (proto project "clojure/protobuf/collections.proto"))) - -(defn proto - ([project] - (if (= "clojure-protobuf" (:name project)) - (build-protobuf project) - (apply proto project (proto-files "proto")))) - ([project & files] - (install) - (let [src-dir (or (:proto-source-dir project) "lib/src")] - (doseq [file files] - (protoc file src-dir)) - (javac {:srcdir (make-path src-dir) - :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))})))) diff --git a/src/test.clj b/src/test.clj deleted file mode 100644 index ff560d4..0000000 --- a/src/test.clj +++ /dev/null @@ -1,14 +0,0 @@ -(use 'clojure.test) - -(def test-names [:protobuf]) - -(def test-namespaces - (map #(symbol (str "test." (name %))) - test-names)) - -(defn run [] - (println "Loading tests...") - (apply require :reload-all test-namespaces) - (apply run-tests test-namespaces)) - -(run) \ No newline at end of file From e4fdac3fbd19f7113832a6d78bae5723ac3ebb6a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 16 Jun 2010 15:09:40 -0700 Subject: [PATCH 053/525] save your fork --- .gitignore | 2 + project.clj | 27 ++----- proto/clojure/protobuf/collections.proto | 2 + src/clj/leiningen/install_protoc.clj | 42 ----------- src/clj/leiningen/proto.clj | 75 ------------------- src/clj/protobuf/tasks.clj | 91 ++++++++++++++++++++++++ src/leiningen | 1 - 7 files changed, 99 insertions(+), 141 deletions(-) delete mode 100644 src/clj/leiningen/install_protoc.clj delete mode 100644 src/clj/leiningen/proto.clj create mode 100644 src/clj/protobuf/tasks.clj delete mode 120000 src/leiningen diff --git a/.gitignore b/.gitignore index a269431..6af1d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +.cake *~ lib classes +build pom.xml *.jar *.class \ No newline at end of file diff --git a/project.clj b/project.clj index 3bf68d6..4af8c2e 100644 --- a/project.clj +++ b/project.clj @@ -1,27 +1,8 @@ (defproject clojure-protobuf "0.1.0-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] - [clojure-contrib "1.2.0-SNAPSHOT"] - [classlojure "0.1.0-SNAPSHOT"]] - :source-path "src/clj" - :java-source-path "src/jvm" - :resources-path "proto") + [clojure-contrib "1.2.0-SNAPSHOT"]] + :tasks [protobuf.tasks] + :jar-files ["proto"]) -(use 'clojure.contrib.with-ns) -(require 'leiningen.compile) -(with-ns 'leiningen.compile - - (defn proto [project] - (try (require 'classlojure) - (require 'leiningen.proto) - ((ns-resolve 'leiningen.proto 'proto) project) - (catch java.io.FileNotFoundException e - (println "you must run 'lein deps' first") - (System/exit 1)))) - - (defn compile [project] - (proto project) - (lancet/javac {:srcdir (make-path (:java-source-path project)) - :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))})) -) \ No newline at end of file +(deftask compile => proto) \ No newline at end of file diff --git a/proto/clojure/protobuf/collections.proto b/proto/clojure/protobuf/collections.proto index 9883606..cfe5c0c 100644 --- a/proto/clojure/protobuf/collections.proto +++ b/proto/clojure/protobuf/collections.proto @@ -7,4 +7,6 @@ extend google.protobuf.FieldOptions { optional bool set = 52001; optional bool map = 52002; optional string map_by = 52003; + // optional bool sorted = 52004; + // optional bool multi = 52005; } diff --git a/src/clj/leiningen/install_protoc.clj b/src/clj/leiningen/install_protoc.clj deleted file mode 100644 index 1d267f1..0000000 --- a/src/clj/leiningen/install_protoc.clj +++ /dev/null @@ -1,42 +0,0 @@ -(ns leiningen.install-protoc - (:use clojure.contrib.shell-out - clojure.contrib.str-utils - lancet)) - -(def proto-version "protobuf-2.3.0") -(def build-dir "lib/build/") -(def protobuf-dir (str build-dir proto-version)) - -(defn shell [& args] - (let [return (apply sh :return-map true args)] - (if (= 0 (return :exit)) - (return :out) - (throw (Exception. (str "command failed: " (str-join " " args) "\n" (return :err))))))) - -(defn exists? [file] - (.exists (java.io.File. file))) - -(deftarget fetch-source "fetch google protocol buffer source" - (let [url (java.net.URL. (str "http://protobuf.googlecode.com/files/" proto-version ".tar.gz")) - tarfile (str protobuf-dir ".tar.gz") - gzip (doto (org.apache.tools.ant.taskdefs.Untar$UntarCompressionMethod.) (.setValue "gzip"))] - (when-not (exists? protobuf-dir) - (mkdir {:dir build-dir}) - (ant-get {:src url :dest tarfile}) - (untar {:src tarfile :dest build-dir :compression gzip})))) - -(deftarget install "install google protocol buffers" - (when (empty? (sh "which" "protoc")) - (fetch-source) - (with-sh-dir protobuf-dir - (when-not (exists? (str protobuf-dir "/src/protoc")) - (println "building google protobuf compiler...") - (chmod {:file (str protobuf-dir "/configure") :perm "+x"}) - (chmod {:file (str protobuf-dir "/install-sh") :perm "+x"}) - (shell "./configure") - (shell "make")) - (println "installing google protobuf compiler...") - (shell "sudo" "make" "install")))) - -(defn install-protoc [project] - (install)) \ No newline at end of file diff --git a/src/clj/leiningen/proto.clj b/src/clj/leiningen/proto.clj deleted file mode 100644 index a53be76..0000000 --- a/src/clj/leiningen/proto.clj +++ /dev/null @@ -1,75 +0,0 @@ -(ns leiningen.proto - (:use leiningen.install-protoc - [leiningen.classpath :only [make-path get-classpath]] - [classlojure :only [extract-resource]] - [clojure.contrib.io :only [file read-lines]] - lancet)) - -(def proto-dir "proto") -(def proto-extract-dir "lib/proto") -(def proto-src-dir "lib/proto_src") - -(defn- match [p s] - (when s - (let [p (if (string? p) (re-pattern p) p)] - (second (re-matches p s))))) - -(defn- proto-exists? [proto] - (or (.exists (file proto-dir proto)) - (.exists (file proto-extract-dir proto)))) - -(defn- proto-dependencies - "look for lines starting with import in proto-file" - [proto-file] - (for [line (read-lines proto-file) :when (.startsWith line "import")] - (match #".*\"(.*)\".*" line))) - -(defn extract-dependencies - "extract all files proto is dependent on" - [proto] - (loop [files (vec (proto-dependencies (file proto-dir proto)))] - (when-not (empty? files) - (let [proto (peek files) - files (pop files)] - (if (proto-exists? proto) - (recur files) - (let [proto-file (extract-resource proto proto-extract-dir)] - (recur (into files (proto-dependencies proto-file))))))))) - -(defn protoc [proto src-dir] - (println "compiling" proto "to" src-dir) - (extract-dependencies proto) - (mkdir {:dir src-dir}) - (shell :dir proto-dir - "protoc" proto (str "--java_out=../" src-dir) "-I." (str "-I../" proto-extract-dir))) - -(declare proto) - -(defn build-protobuf [project] - (fetch-source) - (let [src-dir (str protobuf-dir "/java/src/main/java") - descriptor "google/protobuf/descriptor.proto" - todir (str proto-dir "/google/protobuf")] - (mkdir {:dir todir}) - (copy {:file (str protobuf-dir "/src/" descriptor) :todir todir}) - (protoc descriptor src-dir) - (javac {:srcdir (make-path src-dir) - :destdir (:compile-path project)}) - (proto project "clojure/protobuf/collections.proto"))) - -(defn- proto-files [dir] - (for [file (rest (file-seq (file dir))) :when (re-find #"\.proto$" (.getName file))] - (.substring (.getPath file) (inc (count dir))))) - -(defn proto - ([project] - (if (= "clojure-protobuf" (:name project)) - (build-protobuf project) - (apply proto project (proto-files proto-dir)))) - ([project & files] - (install) - (doseq [file files] - (protoc file proto-src-dir)) - (javac {:srcdir (make-path proto-src-dir) - :destdir (:compile-path project) - :classpath (apply make-path (get-classpath project))}))) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj new file mode 100644 index 0000000..a334c21 --- /dev/null +++ b/src/clj/protobuf/tasks.clj @@ -0,0 +1,91 @@ +(ns protobuf.tasks + (:use cake cake.ant + [clojure.java.shell :only [sh]] + [clojure.java.io :only [copy reader]]) + (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar] + [java.net URL URLClassLoader URLConnection JarURLConnection] + [java.util.jar JarEntry] + [java.io File])) + +(def version "2.3.0") +(def srcdir (format "build/protobuf-%s" version)) +(def tarfile (format "build/protobuf-%s.tar.gz" version)) +(def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) + +(defn installed? [] + (try (.contains (:out (sh "protoc" "--version")) version) + (catch java.io.IOException e))) + +(deftask fetch-protoc + (when-not (.exists (file srcdir)) + (ant Get {:src url :dest tarfile}) + (ant Untar {:src tarfile :dest "build" :compression "gzip"}))) + +(deftask install-protoc + (when-not (installed?) + (run-task 'fetch-protoc) + (when-not (.exists (file srcdir "src" "protoc")) + (ant Chmod {:file (file srcdir "configure") :perm "+x"}) + (ant Chmod {:file (file srcdir "install-sh") :perm "+x"}) + (ant ExecTask {:dir srcdir :executable "./configure"}) + (ant ExecTask {:dir srcdir :executable "make"})) + (ant ExecTask {:dir srcdir :executable "sudo"} + (args ["make" "install"])))) + +(defn- proto-dependencies + "look for lines starting with import in proto-file" + [proto-file] + (for [line (line-seq (reader proto-file)) :when (.startsWith line "import")] + (second (re-matches #".*\"(.*)\".*" line)))) + +(defn extract-resource [name dest-dir] + (let [loader (.getClassLoader clojure.lang.RT) + url (.findResource loader name) + conn (.openConnection url)] + (if (instance? JarURLConnection conn) + (let [jar (cast JarURLConnection conn) + dest (File. dest-dir (.. jar getJarEntry getName))] + (copy (.getInputStream jar) dest)) + (copy (File. (.getFile url)) (File. dest-dir name))))) + +(defn extract-dependencies + "extract all files proto is dependent on" + [proto-file] + (loop [files (vec (proto-dependencies proto-file))] + (when-not (empty? files) + (let [proto (peek files) + files (pop files)] + (if (or (.exists (file "proto" proto)) (.exists (file "build/proto" proto))) + (recur files) + (let [proto-file (extract-resource proto "build/proto")] + (recur (into files (proto-dependencies proto-file))))))))) + +(defn protoc + ([protos] (protoc protos "build/protosrc")) + ([protos dest] + (doseq [proto protos] + (log "compiling" proto "to" dest) + (extract-dependencies (file "proto" proto)) + (ant Mkdir {:dir dest}) + (ant Mkdir {:dir "build/proto"}) + (ant ExecTask {:executable "protoc" :dir "proto"} + (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"]))) + (ant Javac {:srcdir (path dest) + :destdir (file "classes") + :classpath (classpath project)}))) + +(defn build-protobuf [] + (ant Mkdir {:dir "proto/google/protobuf"}) + (ant Copy {:file (str srcdir "/src/google/protobuf/descriptor.proto") :todir "proto/google/protobuf"}) + (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java")) + (protoc ["clojure/protobuf/collections.proto"])) + +(defn proto-files [dir] + (for [file (rest (file-seq dir)) :when (.endsWith (.getName file) ".proto")] + (.substring (.getPath file) (inc (count (.getPath dir)))))) + +(deftask proto + (if (= "clojure-protobuf" (:artifact-id project)) + (do (run-task 'fetch-protoc) + (build-protobuf)) + (protoc (or (:proto opts) (proto-files (file "proto")))))) diff --git a/src/leiningen b/src/leiningen deleted file mode 120000 index 369c96c..0000000 --- a/src/leiningen +++ /dev/null @@ -1 +0,0 @@ -clj/leiningen \ No newline at end of file From d39b59a5015be620ea8e34d17dca0f46d3fdaf23 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 16 Jun 2010 15:10:14 -0700 Subject: [PATCH 054/525] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4af8c2e..ed97ab1 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.1.0-SNAPSHOT" +(defproject clojure-protobuf "0.2.0" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] From 373502f894917e72513e5893123980d507c43e7a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 16 Jun 2010 17:11:59 -0700 Subject: [PATCH 055/525] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index ed97ab1..ea97e2f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.0" +(defproject clojure-protobuf "0.2.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] From 7785e7a5e4e2283f305c12670059c1ba8b13e056 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 16 Jun 2010 17:48:22 -0700 Subject: [PATCH 056/525] improved error message when proto files are missing --- src/clj/protobuf/tasks.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index a334c21..1d86f0b 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -39,14 +39,14 @@ (second (re-matches #".*\"(.*)\".*" line)))) (defn extract-resource [name dest-dir] - (let [loader (.getClassLoader clojure.lang.RT) - url (.findResource loader name) - conn (.openConnection url)] - (if (instance? JarURLConnection conn) - (let [jar (cast JarURLConnection conn) - dest (File. dest-dir (.. jar getJarEntry getName))] - (copy (.getInputStream jar) dest)) - (copy (File. (.getFile url)) (File. dest-dir name))))) + (if-let [url (.findResource (.getClassLoader clojure.lang.RT) name)] + (let [conn (.openConnection url)] + (if (instance? JarURLConnection conn) + (let [jar (cast JarURLConnection conn) + dest (File. dest-dir (.. jar getJarEntry getName))] + (copy (.getInputStream jar) dest)) + (copy (File. (.getFile url)) (File. dest-dir name)))) + (throw (Exception. (format "unable to find %s on classpath" name))))) (defn extract-dependencies "extract all files proto is dependent on" From cac29903c85c757d9061e524e5660952d6f44b1b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 17 Jun 2010 11:09:19 -0700 Subject: [PATCH 057/525] move .gitignore out of proto dir --- .gitignore | 3 ++- proto/.gitignore | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 proto/.gitignore diff --git a/.gitignore b/.gitignore index 6af1d4d..79b5bdd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ classes build pom.xml *.jar -*.class \ No newline at end of file +*.class +proto/google/protobuf/descriptor.proto diff --git a/proto/.gitignore b/proto/.gitignore deleted file mode 100644 index 576998b..0000000 --- a/proto/.gitignore +++ /dev/null @@ -1 +0,0 @@ -google/protobuf/descriptor.proto \ No newline at end of file From 78677d4d4f0ff4b602607dda444fcdc85b2a1a6d Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 17 Jun 2010 15:49:11 -0700 Subject: [PATCH 058/525] extract proto files from the proto dir --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 1d86f0b..83dcde9 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -57,7 +57,7 @@ files (pop files)] (if (or (.exists (file "proto" proto)) (.exists (file "build/proto" proto))) (recur files) - (let [proto-file (extract-resource proto "build/proto")] + (let [proto-file (extract-resource (str "proto/" proto) "build/proto")] (recur (into files (proto-dependencies proto-file))))))))) (defn protoc From aea9e6b6632ba1068c34c475ebcab73bd12d2c89 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 17 Jun 2010 16:29:37 -0700 Subject: [PATCH 059/525] fix extract-dependencies in tasks --- src/clj/protobuf/tasks.clj | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 83dcde9..d2c089d 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -32,24 +32,24 @@ (ant ExecTask {:dir srcdir :executable "sudo"} (args ["make" "install"])))) -(defn- proto-dependencies - "look for lines starting with import in proto-file" +(defn- proto-dependencies "look for lines starting with import in proto-file" [proto-file] (for [line (line-seq (reader proto-file)) :when (.startsWith line "import")] (second (re-matches #".*\"(.*)\".*" line)))) (defn extract-resource [name dest-dir] (if-let [url (.findResource (.getClassLoader clojure.lang.RT) name)] - (let [conn (.openConnection url)] + (let [dest (File. dest-dir name) + conn (.openConnection url)] + (.mkdirs (.getParentFile dest)) (if (instance? JarURLConnection conn) - (let [jar (cast JarURLConnection conn) - dest (File. dest-dir (.. jar getJarEntry getName))] + (let [jar (cast JarURLConnection conn)] (copy (.getInputStream jar) dest)) - (copy (File. (.getFile url)) (File. dest-dir name)))) + (copy (File. (.getFile url)) dest)) + dest) (throw (Exception. (format "unable to find %s on classpath" name))))) -(defn extract-dependencies - "extract all files proto is dependent on" +(defn extract-dependencies "extract all files proto is dependent on" [proto-file] (loop [files (vec (proto-dependencies proto-file))] (when-not (empty? files) @@ -57,7 +57,7 @@ files (pop files)] (if (or (.exists (file "proto" proto)) (.exists (file "build/proto" proto))) (recur files) - (let [proto-file (extract-resource (str "proto/" proto) "build/proto")] + (let [proto-file (extract-resource (str "proto/" proto) "build")] (recur (into files (proto-dependencies proto-file))))))))) (defn protoc From 456c9b0c3aaa17fb401766761cbb8406bb217908 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 21 Jun 2010 16:40:08 -0700 Subject: [PATCH 060/525] build protos before compiling --- src/clj/protobuf/tasks.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index d2c089d..4d2fb1d 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -84,6 +84,7 @@ (for [file (rest (file-seq dir)) :when (.endsWith (.getName file) ".proto")] (.substring (.getPath file) (inc (count (.getPath dir)))))) +(deftask compile #{proto}) (deftask proto (if (= "clojure-protobuf" (:artifact-id project)) (do (run-task 'fetch-protoc) From 94e7772a0590fa2a3bf13b7184266542b1b24784 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 29 Jun 2010 11:40:21 -0700 Subject: [PATCH 061/525] add license --- LICENSE | 227 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fec593a --- /dev/null +++ b/LICENSE @@ -0,0 +1,227 @@ +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE +PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF +THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and +documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and +are distributed by that particular Contributor. A Contribution +'originates' from a Contributor if it was added to the Program by such +Contributor itself or anyone acting on such Contributor's +behalf. Contributions do not include additions to the Program which: +(i) are separate modules of software distributed in conjunction with +the Program under their own license agreement, and (ii) are not +derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor +which are necessarily infringed by the use or sale of its Contribution +alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this +Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby +grants Recipient a non-exclusive, worldwide, royalty-free copyright +license to reproduce, prepare derivative works of, publicly display, +publicly perform, distribute and sublicense the Contribution of such +Contributor, if any, and such derivative works, in source code and +object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby +grants Recipient a non-exclusive, worldwide, royalty-free patent +license under Licensed Patents to make, use, sell, offer to sell, +import and otherwise transfer the Contribution of such Contributor, if +any, in source code and object code form. This patent license shall +apply to the combination of the Contribution and the Program if, at +the time the Contribution is added by the Contributor, such addition +of the Contribution causes such combination to be covered by the +Licensed Patents. The patent license shall not apply to any other +combinations which include the Contribution. No hardware per se is +licensed hereunder. + +c) Recipient understands that although each Contributor grants the +licenses to its Contributions set forth herein, no assurances are +provided by any Contributor that the Program does not infringe the +patent or other intellectual property rights of any other entity. Each +Contributor disclaims any liability to Recipient for claims brought by +any other entity based on infringement of intellectual property rights +or otherwise. As a condition to exercising the rights and licenses +granted hereunder, each Recipient hereby assumes sole responsibility +to secure any other intellectual property rights needed, if any. For +example, if a third party patent license is required to allow +Recipient to distribute the Program, it is Recipient's responsibility +to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient +copyright rights in its Contribution, if any, to grant the copyright +license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form +under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties +and conditions, express and implied, including warranties or +conditions of title and non-infringement, and implied warranties or +conditions of merchantability and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability +for damages, including direct, indirect, special, incidental and +consequential damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are +offered by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such +Contributor, and informs licensees how to obtain it in a reasonable +manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained +within the Program. + +Each Contributor must identify itself as the originator of its +Contribution, if any, in a manner that reasonably allows subsequent +Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain +responsibilities with respect to end users, business partners and the +like. While this license is intended to facilitate the commercial use +of the Program, the Contributor who includes the Program in a +commercial product offering should do so in a manner which does not +create potential liability for other Contributors. Therefore, if a +Contributor includes the Program in a commercial product offering, +such Contributor ("Commercial Contributor") hereby agrees to defend +and indemnify every other Contributor ("Indemnified Contributor") +against any losses, damages and costs (collectively "Losses") arising +from claims, lawsuits and other legal actions brought by a third party +against the Indemnified Contributor to the extent caused by the acts +or omissions of such Commercial Contributor in connection with its +distribution of the Program in a commercial product offering. The +obligations in this section do not apply to any claims or Losses +relating to any actual or alleged intellectual property +infringement. In order to qualify, an Indemnified Contributor must: a) +promptly notify the Commercial Contributor in writing of such claim, +and b) allow the Commercial Contributor tocontrol, and cooperate with +the Commercial Contributor in, the defense and any related settlement +negotiations. The Indemnified Contributor may participate in any such +claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those +performance claims and warranties, and if a court requires any other +Contributor to pay any damages as a result, the Commercial Contributor +must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY +WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY +OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely +responsible for determining the appropriateness of using and +distributing the Program and assumes all risks associated with its +exercise of rights under this Agreement , including but not limited to +the risks and costs of program errors, compliance with applicable +laws, damage to or loss of data, programs or equipment, and +unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR +ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING +WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR +DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED +HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that +the Program itself (excluding combinations of the Program with other +software or hardware) infringes such Recipient's patent(s), then such +Recipient's rights granted under Section 2(b) shall terminate as of +the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably +practicable. However, Recipient's obligations under this Agreement and +any licenses granted by Recipient relating to the Program shall +continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign +the responsibility to serve as the Agreement Steward to a suitable +separate entity. Each new version of the Agreement will be given a +distinguishing version number. The Program (including Contributions) +may always be distributed subject to the version of the Agreement +under which it was received. In addition, after a new version of the +Agreement is published, Contributor may elect to distribute the +Program (including its Contributions) under the new version. Except as +expressly stated in Sections 2(a) and 2(b) above, Recipient receives +no rights or licenses to the intellectual property of any Contributor +under this Agreement, whether expressly, by implication, estoppel or +otherwise. All rights in the Program not expressly granted under this +Agreement are reserved. + +This Agreement is governed by the laws of the State of Washington and +the intellectual property laws of the United States of America. No +party to this Agreement will bring a legal action under this Agreement +more than one year after the cause of action arose. Each party waives +its rights to a jury trial in any resulting litigation. From aa6fdce48340774352f71a42e67f6fe6748867ac Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 29 Jun 2010 11:40:46 -0700 Subject: [PATCH 062/525] use useful.io/extract-resource --- project.clj | 4 +--- src/clj/protobuf/tasks.clj | 20 ++++---------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/project.clj b/project.clj index ea97e2f..4631598 100644 --- a/project.clj +++ b/project.clj @@ -3,6 +3,4 @@ :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] :tasks [protobuf.tasks] - :jar-files ["proto"]) - -(deftask compile => proto) \ No newline at end of file + :jar-files ["proto"]) \ No newline at end of file diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 4d2fb1d..05ce49d 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,16 +1,16 @@ (ns protobuf.tasks (:use cake cake.ant [clojure.java.shell :only [sh]] - [clojure.java.io :only [copy reader]]) + [clojure.java.io :only [reader]] + [useful.io :only [extract-resource]]) (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar] - [java.net URL URLClassLoader URLConnection JarURLConnection] - [java.util.jar JarEntry] + [java.net URL URLConnection JarURLConnection] [java.io File])) (def version "2.3.0") (def srcdir (format "build/protobuf-%s" version)) (def tarfile (format "build/protobuf-%s.tar.gz" version)) -(def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) +(def url (URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) (defn installed? [] (try (.contains (:out (sh "protoc" "--version")) version) @@ -37,18 +37,6 @@ (for [line (line-seq (reader proto-file)) :when (.startsWith line "import")] (second (re-matches #".*\"(.*)\".*" line)))) -(defn extract-resource [name dest-dir] - (if-let [url (.findResource (.getClassLoader clojure.lang.RT) name)] - (let [dest (File. dest-dir name) - conn (.openConnection url)] - (.mkdirs (.getParentFile dest)) - (if (instance? JarURLConnection conn) - (let [jar (cast JarURLConnection conn)] - (copy (.getInputStream jar) dest)) - (copy (File. (.getFile url)) dest)) - dest) - (throw (Exception. (format "unable to find %s on classpath" name))))) - (defn extract-dependencies "extract all files proto is dependent on" [proto-file] (loop [files (vec (proto-dependencies proto-file))] From 1bfcb9e752023d38f39b6348af0a887740e98021 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 29 Jun 2010 11:41:36 -0700 Subject: [PATCH 063/525] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4631598..e11da5a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.1-SNAPSHOT" +(defproject clojure-protobuf "0.2.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] From 3bcf47f19f64ac192c4096e9920d77e95fc6537a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 30 Jun 2010 11:22:46 -0700 Subject: [PATCH 064/525] don't rebuild protos unless the files have changed --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/project.clj b/project.clj index e11da5a..278d8d0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.1" +(defproject clojure-protobuf "0.2.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 05ce49d..824223e 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -3,14 +3,12 @@ [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] [useful.io :only [extract-resource]]) - (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar] - [java.net URL URLConnection JarURLConnection] - [java.io File])) + (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") (def srcdir (format "build/protobuf-%s" version)) (def tarfile (format "build/protobuf-%s.tar.gz" version)) -(def url (URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) +(def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) (defn installed? [] (try (.contains (:out (sh "protoc" "--version")) version) @@ -48,19 +46,23 @@ (let [proto-file (extract-resource (str "proto/" proto) "build")] (recur (into files (proto-dependencies proto-file))))))))) +(defn modtime [dir] + (apply max (map #(.lastModified %) (file-seq (file dir))))) + (defn protoc ([protos] (protoc protos "build/protosrc")) ([protos dest] - (doseq [proto protos] - (log "compiling" proto "to" dest) - (extract-dependencies (file "proto" proto)) - (ant Mkdir {:dir dest}) - (ant Mkdir {:dir "build/proto"}) - (ant ExecTask {:executable "protoc" :dir "proto"} - (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"]))) - (ant Javac {:srcdir (path dest) - :destdir (file "classes") - :classpath (classpath project)}))) + (when (> (modtime "proto") (modtime dest)) + (doseq [proto protos] + (log "compiling" proto "to" dest) + (extract-dependencies (file "proto" proto)) + (ant Mkdir {:dir dest}) + (ant Mkdir {:dir "build/proto"}) + (ant ExecTask {:executable "protoc" :dir "proto"} + (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"]))) + (ant Javac {:srcdir (path dest) + :destdir (file "classes") + :classpath (classpath project)})))) (defn build-protobuf [] (ant Mkdir {:dir "proto/google/protobuf"}) From 510bdca82dce094ccbcc4997b80d3c4d4ae8db2c Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 6 Jul 2010 15:13:25 -0700 Subject: [PATCH 065/525] install protoc before trying to compile proto files --- src/clj/protobuf/tasks.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 824223e..fca5a41 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -75,7 +75,8 @@ (.substring (.getPath file) (inc (count (.getPath dir)))))) (deftask compile #{proto}) -(deftask proto +(deftask proto #{install-protoc} + "Compile protocol buffer files located in proto dir." (if (= "clojure-protobuf" (:artifact-id project)) (do (run-task 'fetch-protoc) (build-protobuf)) From 54f6a02d84a1bdb59691a14f05cc5c057615ab4c Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 6 Jul 2010 16:03:02 -0700 Subject: [PATCH 066/525] recompile protobufs if src or class dir is older than proto dir. --- src/clj/protobuf/tasks.clj | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index fca5a41..4610134 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -47,12 +47,16 @@ (recur (into files (proto-dependencies proto-file))))))))) (defn modtime [dir] - (apply max (map #(.lastModified %) (file-seq (file dir))))) + (let [files (rest (file-seq (file dir)))] + (if (empty? files) + 0 + (apply max (map #(.lastModified %) files))))) (defn protoc ([protos] (protoc protos "build/protosrc")) ([protos dest] - (when (> (modtime "proto") (modtime dest)) + (when (or (> (modtime "proto") (modtime dest)) + (> (modtime "proto") (modtime "classes"))) (doseq [proto protos] (log "compiling" proto "to" dest) (extract-dependencies (file "proto" proto)) From 96a0ced8df31ef6d8e7554b0ae309d1f32a317fb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 6 Jul 2010 16:03:23 -0700 Subject: [PATCH 067/525] bump version --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 278d8d0..693b3d9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.2" +(defproject clojure-protobuf "0.2.3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] From 70e37db310548200d615b075a61392deb2c32043 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 19 Jul 2010 13:09:51 -0700 Subject: [PATCH 068/525] fix tests to run with cake --- {test => proto/clojure/protobuf}/test.proto | 4 ++-- src/clj/protobuf.clj | 4 ++-- src/clj/protobuf/tasks.clj | 12 +++++++++--- test/{protobuf.clj => protobuf_test.clj} | 8 ++++---- 4 files changed, 17 insertions(+), 11 deletions(-) rename {test => proto/clojure/protobuf}/test.proto (89%) rename test/{protobuf.clj => protobuf_test.clj} (95%) diff --git a/test/test.proto b/proto/clojure/protobuf/test.proto similarity index 89% rename from test/test.proto rename to proto/clojure/protobuf/test.proto index 419cc11..583fec3 100644 --- a/test/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -1,7 +1,7 @@ -package test.protobuf; import "clojure/protobuf/collections.proto"; -option java_outer_classname = "Proto"; +option java_package = "clojure.protobuf"; +option java_outer_classname = "Test"; message Foo { optional uint32 id = 1 [default = 43]; diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index ee73d55..98e717b 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -30,8 +30,8 @@ (defn protofields [type] (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type) type #^com.google.protobuf.Descriptors$Descriptor (.getMessageType type)] - (map (fn [#^com.google.protobuf.Descriptors$FieldDescriptor field] (keyword (.getName field))) - (.getFields type)))) + (for [#^com.google.protobuf.Descriptors$FieldDescriptor field (.getFields type)] + (keyword (.getName field))))) (defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 4610134..43d3289 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -55,7 +55,8 @@ (defn protoc ([protos] (protoc protos "build/protosrc")) ([protos dest] - (when (or (> (modtime "proto") (modtime dest)) + (when (or (:recompile opts) + (> (modtime "proto") (modtime dest)) (> (modtime "proto") (modtime "classes"))) (doseq [proto protos] (log "compiling" proto "to" dest) @@ -72,10 +73,15 @@ (ant Mkdir {:dir "proto/google/protobuf"}) (ant Copy {:file (str srcdir "/src/google/protobuf/descriptor.proto") :todir "proto/google/protobuf"}) (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java")) - (protoc ["clojure/protobuf/collections.proto"])) + (protoc ["clojure/protobuf/collections.proto" "clojure/protobuf/test.proto"])) + +(defn proto-file? [file] + (let [name (.getName file)] + (and (.endsWith name ".proto") + (not (.startsWith name "."))))) (defn proto-files [dir] - (for [file (rest (file-seq dir)) :when (.endsWith (.getName file) ".proto")] + (for [file (rest (file-seq dir)) :when (proto-file? file)] (.substring (.getPath file) (inc (count (.getPath dir)))))) (deftask compile #{proto}) diff --git a/test/protobuf.clj b/test/protobuf_test.clj similarity index 95% rename from test/protobuf.clj rename to test/protobuf_test.clj index 38360a0..9959b45 100644 --- a/test/protobuf.clj +++ b/test/protobuf_test.clj @@ -1,8 +1,8 @@ -(ns test.protobuf +(ns protobuf-test (:use protobuf) (:use clojure.test)) -(defprotobuf Foo test.protobuf.Proto Foo) +(defprotobuf Foo clojure.protobuf.Test Foo) (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) @@ -62,7 +62,7 @@ (testing "protofields" (let [fields '(:id :label :tags :parent :responses :tag_set :attr_map :foo_by_id :groups)] (is (= fields (protofields Foo))) - (is (= fields (protofields test.protobuf.Proto$Foo))))) + (is (= fields (protofields clojure.protobuf.Test$Foo))))) (testing "protodefault" (is (= 43 (protodefault Foo :id))) (is (= "" (protodefault Foo :label))) @@ -72,7 +72,7 @@ (is (= #{} (protodefault Foo :tag_set))) (is (= {} (protodefault Foo :foo_by_id))) (is (= {} (protodefault Foo :groups))) - (is (= {} (protodefault test.protobuf.Proto$Foo :groups))) + (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))) )) (deftest protobuf-extended From 8ee26d50fc65a4135f7d23fb880439b82d8f92f1 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 19 Jul 2010 13:48:51 -0700 Subject: [PATCH 069/525] rename collections.proto to extensions.proto. add meta extension field --- project.clj | 2 +- .../protobuf/{collections.proto => extensions.proto} | 3 ++- proto/clojure/protobuf/test.proto | 4 ++-- src/clj/protobuf.clj | 6 ++++-- src/clj/protobuf/tasks.clj | 2 +- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 8 ++++---- test/protobuf_test.clj | 3 ++- 7 files changed, 16 insertions(+), 12 deletions(-) rename proto/clojure/protobuf/{collections.proto => extensions.proto} (79%) diff --git a/project.clj b/project.clj index 693b3d9..46ccd22 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.3" +(defproject clojure-protobuf "0.2.4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] diff --git a/proto/clojure/protobuf/collections.proto b/proto/clojure/protobuf/extensions.proto similarity index 79% rename from proto/clojure/protobuf/collections.proto rename to proto/clojure/protobuf/extensions.proto index cfe5c0c..4441d4f 100644 --- a/proto/clojure/protobuf/collections.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -1,7 +1,7 @@ import "google/protobuf/descriptor.proto"; option java_package = "clojure.protobuf"; -option java_outer_classname = "Collections"; +option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { optional bool set = 52001; @@ -9,4 +9,5 @@ extend google.protobuf.FieldOptions { optional string map_by = 52003; // optional bool sorted = 52004; // optional bool multi = 52005; + optional string meta = 52006; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 583fec3..edab84c 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -1,11 +1,11 @@ -import "clojure/protobuf/collections.proto"; +import "clojure/protobuf/extensions.proto"; option java_package = "clojure.protobuf"; option java_outer_classname = "Test"; message Foo { optional uint32 id = 1 [default = 43]; - optional string label = 2; + optional string label = 2 [(meta) = "{:a 1 :b 2 :c 3}"]; repeated string tags = 3; optional Foo parent = 4; repeated Response responses = 5; diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 98e717b..fbd0f66 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -30,8 +30,10 @@ (defn protofields [type] (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type) type #^com.google.protobuf.Descriptors$Descriptor (.getMessageType type)] - (for [#^com.google.protobuf.Descriptors$FieldDescriptor field (.getFields type)] - (keyword (.getName field))))) + (into {} + (for [#^com.google.protobuf.Descriptors$FieldDescriptor field (.getFields type)] + (let [meta-string (.. field getOptions (getExtension (clojure.protobuf.Extensions/meta)))] + [(keyword (.getName field)) (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 43d3289..a4dfd80 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -73,7 +73,7 @@ (ant Mkdir {:dir "proto/google/protobuf"}) (ant Copy {:file (str srcdir "/src/google/protobuf/descriptor.proto") :todir "proto/google/protobuf"}) (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java")) - (protoc ["clojure/protobuf/collections.proto" "clojure/protobuf/test.proto"])) + (protoc ["clojure/protobuf/extensions.proto" "clojure/protobuf/test.proto"])) (defn proto-file? [file] (let [name (.getName file)] diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index cc210c1..49fa069 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -190,7 +190,7 @@ static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_valu static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { Keyword keyword = map_field_by.get(field); if (keyword == null) { - String name = field.getOptions().getExtension(Collections.mapBy); + String name = field.getOptions().getExtension(Extensions.mapBy); name = name.toLowerCase().replaceAll("_","-"); keyword = Keyword.intern(Symbol.intern(name)); @@ -221,7 +221,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } return map.persistent(); - } else if (options.getExtension(Collections.map)) { + } else if (options.getExtension(Extensions.map)) { Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); Descriptors.FieldDescriptor val_field = def.fieldDescriptor(k_val); @@ -244,7 +244,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object while (iterator.hasNext()) { list.add(fromProtoValue(field, iterator.next())); } - return options.getExtension(Collections.set) ? PersistentHashSet.create(list) : PersistentVector.create(list); + return options.getExtension(Extensions.set) ? PersistentHashSet.create(list) : PersistentVector.create(list); } } else { switch (field.getJavaType()) { @@ -318,7 +318,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) Object value = toProtoValue(field, map.assoc(map_field_by, e.getKey())); builder.addRepeatedField(field, value); } - } else if (field.getOptions().getExtension(Collections.map)) { + } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(val); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 9959b45..77d93af 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -60,7 +60,8 @@ (is (= ["sweet" "savory"] (s :tags))) )) (testing "protofields" - (let [fields '(:id :label :tags :parent :responses :tag_set :attr_map :foo_by_id :groups)] + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, + :tag_set nil, :attr_map nil, :foo_by_id nil, :groups nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) (testing "protodefault" From c81fbe51e39f3e3825e1ae34efabadfdbe6e9262 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 19 Jul 2010 14:24:40 -0700 Subject: [PATCH 070/525] update readme --- README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 42f908f..c3d893d 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ Write a `.proto` file: repeated string likes = 4; } -If you put it in the protos directory of your project, you can compile it with lein: +If you put it in the proto directory of your project, you can compile it with cake: - lein proto example.proto + cake proto example.proto Now you can use the protocol buffer in clojure: @@ -40,12 +40,13 @@ Now you can use the protocol buffer in clojure: A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. -## Collection extensions +## Extensions Clojure-protobuf supports extensions to protocol buffers which provide sets and maps using -repeated fields. To use these, you must import the extension file and include it when compiling. For example: +repeated fields. You can also provide metadata on protobuf fields using clojure syntax. To +use these, you must import the extension file and include it when compiling. For example: - import "collections.proto"; + import "clojure/protobuf/extensions.proto"; message Photo { required int32 id = 1; required string path = 2; @@ -60,7 +61,7 @@ repeated fields. To use these, you must import the extension file and include it message Tag { required int32 person_id = 1; - optional int32 x_coord = 2; + optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; optional int32 y_coord = 3; optional int32 width = 4; optional int32 height = 5; @@ -69,12 +70,13 @@ repeated fields. To use these, you must import the extension file and include it Compile the file: - lein proto example.proto + cake proto example.proto Then you can access the maps in clojure: (use 'protobuf) (defprotobuf Photo Example$Photo) + (defprotobuf Tag Example$Photo$Tag) (def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} @@ -87,15 +89,14 @@ Then you can access the maps in clojure: (protobuf-load Photo b) => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} -## Installation - -In your [Leiningen](http://github.com/technomancy/leiningen) project.clj: + (:x-coord (protofields Tag)) + => {:max 100.0 :min -100.0} - :dependencies [[clojure-protobuf "0.1.0-SNAPSHOT"]] +## Installation -Build from source: +Add clojure-protobuf to your [Cake](http://github.com/ninjudd/cake) project.clj: - lein deps - lein jar + :dependencies [[clojure-protobuf "0.2.4"]] + :dev-dependencies [[clojure-protobuf "0.2.4"]] This code works with Clojure 1.2. Here's the [1.1 branch](http://github.com/ninjudd/clojure-protobuf/tree/clojure-1.1). \ No newline at end of file From 36a42c337f94c57fbcf6ab255b8fb1a9c553d943 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 19 Jul 2010 16:04:51 -0700 Subject: [PATCH 071/525] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c3d893d..2989334 100644 --- a/README.md +++ b/README.md @@ -94,9 +94,10 @@ Then you can access the maps in clojure: ## Installation -Add clojure-protobuf to your [Cake](http://github.com/ninjudd/cake) project.clj: +Add the following to your [Cake](http://github.com/ninjudd/cake) project.clj: :dependencies [[clojure-protobuf "0.2.4"]] :dev-dependencies [[clojure-protobuf "0.2.4"]] + :tasks [protobuf.tasks] This code works with Clojure 1.2. Here's the [1.1 branch](http://github.com/ninjudd/clojure-protobuf/tree/clojure-1.1). \ No newline at end of file From f1d86eacd7072ed5dbb02ea5b289795688293340 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 4 Aug 2010 13:12:35 -0700 Subject: [PATCH 072/525] fix classpath call --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 46ccd22..cd01ab3 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.4" +(defproject clojure-protobuf "0.2.5" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index a4dfd80..c2a1bb0 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -59,7 +59,7 @@ (> (modtime "proto") (modtime dest)) (> (modtime "proto") (modtime "classes"))) (doseq [proto protos] - (log "compiling" proto "to" dest) + (log "Compiling" proto "to" dest) (extract-dependencies (file "proto" proto)) (ant Mkdir {:dir dest}) (ant Mkdir {:dir "build/proto"}) @@ -67,7 +67,7 @@ (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"]))) (ant Javac {:srcdir (path dest) :destdir (file "classes") - :classpath (classpath project)})))) + :classpath (classpath)})))) (defn build-protobuf [] (ant Mkdir {:dir "proto/google/protobuf"}) From c209dcd2b30f7a4e9e444d70c3092bd7f99239be Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 4 Aug 2010 17:34:25 -0700 Subject: [PATCH 073/525] *earmuffs* --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index cd01ab3..c1c6b1c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.5" +(defproject clojure-protobuf "0.2.6" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index c2a1bb0..79207d9 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -55,7 +55,7 @@ (defn protoc ([protos] (protoc protos "build/protosrc")) ([protos dest] - (when (or (:recompile opts) + (when (or (:recompile *opts*) (> (modtime "proto") (modtime dest)) (> (modtime "proto") (modtime "classes"))) (doseq [proto protos] @@ -87,7 +87,7 @@ (deftask compile #{proto}) (deftask proto #{install-protoc} "Compile protocol buffer files located in proto dir." - (if (= "clojure-protobuf" (:artifact-id project)) + (if (= "clojure-protobuf" (:artifact-id *project*)) (do (run-task 'fetch-protoc) (build-protobuf)) - (protoc (or (:proto opts) (proto-files (file "proto")))))) + (protoc (or (:proto *opts*) (proto-files (file "proto")))))) From a1614c63e44b4ed8404f287e41dd455812f86927 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 17 Aug 2010 19:12:48 -0700 Subject: [PATCH 074/525] update to work with cake 0.4.0 --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 79207d9..4d541a0 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,5 +1,5 @@ (ns protobuf.tasks - (:use cake cake.ant + (:use cake cake.core cake.ant [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] [useful.io :only [extract-resource]]) From 4cd8f807366640a89b4cf732a42397996771eadb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 17 Aug 2010 19:13:12 -0700 Subject: [PATCH 075/525] version 0.2.7 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c1c6b1c..cf708ff 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.6" +(defproject clojure-protobuf "0.2.7" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0-master-SNAPSHOT"] [clojure-contrib "1.2.0-SNAPSHOT"]] From 77dc2892af5c9fac2d347060caeda47fb3651274 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 18 Aug 2010 21:22:49 -0700 Subject: [PATCH 076/525] delete build/protosrc when there is an error in protoc --- src/clj/protobuf/tasks.clj | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 4d541a0..645fbba 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -3,7 +3,7 @@ [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] [useful.io :only [extract-resource]]) - (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) + (:import [org.apache.tools.ant.taskdefs Chmod Copy Delete ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") (def srcdir (format "build/protobuf-%s" version)) @@ -57,14 +57,17 @@ ([protos dest] (when (or (:recompile *opts*) (> (modtime "proto") (modtime dest)) - (> (modtime "proto") (modtime "classes"))) + (> (modtime "proto") (modtime "classes"))) + (ant Mkdir {:dir dest}) + (ant Mkdir {:dir "build/proto"}) (doseq [proto protos] (log "Compiling" proto "to" dest) (extract-dependencies (file "proto" proto)) - (ant Mkdir {:dir dest}) - (ant Mkdir {:dir "build/proto"}) - (ant ExecTask {:executable "protoc" :dir "proto"} - (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"]))) + (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} + (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) + (catch org.apache.tools.ant.BuildException e + (ant Delete {:dir dest}) + (throw (Exception. (str "error compiling " proto)))))) (ant Javac {:srcdir (path dest) :destdir (file "classes") :classpath (classpath)})))) From 38214e2199e2868420ce808c85ccba6be84f856e Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 18 Aug 2010 21:23:29 -0700 Subject: [PATCH 077/525] support deletions on append with set extension --- proto/clojure/protobuf/extensions.proto | 8 +-- proto/clojure/protobuf/test.proto | 7 ++- .../protobuf/PersistentProtocolBufferMap.java | 50 +++++++++++++++++-- test/protobuf_test.clj | 8 +-- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/proto/clojure/protobuf/extensions.proto b/proto/clojure/protobuf/extensions.proto index 4441d4f..db4908e 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -4,10 +4,10 @@ option java_package = "clojure.protobuf"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { - optional bool set = 52001; + optional bool set = 52001; optional bool map = 52002; optional string map_by = 52003; - // optional bool sorted = 52004; - // optional bool multi = 52005; - optional string meta = 52006; + // optional bool sorted = 52005; + // optional bool multi = 52006; + optional string meta = 52010; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index edab84c..dd337ef 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -16,7 +16,7 @@ message Foo { MAYBE = 1; } - repeated string tag_set = 10 [(set) = true]; + repeated Item tag_set = 10 [(set) = true]; repeated Pair attr_map = 11 [(map) = true]; repeated Foo foo_by_id = 12 [(map_by) = "id"]; repeated Group groups = 13 [(map) = true]; @@ -31,3 +31,8 @@ message Group { required string key = 1; repeated Foo val = 2; } + +message Item { + required string item = 1; + required bool exists = 2 [default = true]; +} diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 49fa069..85822ba 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -199,8 +199,10 @@ static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { return keyword == k_null ? null : keyword; } - static Keyword k_key = Keyword.intern(Symbol.intern("key")); - static Keyword k_val = Keyword.intern(Symbol.intern("val")); + static Keyword k_key = Keyword.intern(Symbol.intern("key")); + static Keyword k_val = Keyword.intern(Symbol.intern("val")); + static Keyword k_item = Keyword.intern(Symbol.intern("item")); + static Keyword k_exists = Keyword.intern(Symbol.intern("exists")); static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { if (value instanceof List) { List values = (List) value; @@ -239,12 +241,34 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } return map.persistent(); + } else if (options.getExtension(Extensions.set)) { + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); + Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); + + ITransientSet set = (ITransientSet) PersistentHashSet.EMPTY.asTransient(); + while (iterator.hasNext()) { + DynamicMessage message = (DynamicMessage) iterator.next(); + Object item = fromProtoValue(item_field, message.getField(item_field)); + Boolean exists = (Boolean) message.getField(exists_field); + + if (exists) { + set.conj(item); + } else { + try { + set.disjoin(item); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + return set.persistent(); } else { List list = new ArrayList(values.size()); while (iterator.hasNext()) { list.add(fromProtoValue(field, iterator.next())); } - return options.getExtension(Extensions.set) ? PersistentHashSet.create(list) : PersistentVector.create(list); + return PersistentVector.create(list); } } else { switch (field.getJavaType()) { @@ -302,9 +326,10 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v protected void addField(DynamicMessage.Builder builder, Object key, Object val) { if (key == null) return; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { - if (val instanceof Sequential || val instanceof IPersistentSet) { + if (val instanceof Sequential && !set) { for (ISeq s = RT.seq(val); s != null; s = s.next()) { Object value = toProtoValue(field, s.first()); builder.addRepeatedField(field, value); @@ -325,6 +350,21 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) Object value = toProtoValue(field, new PersistentArrayMap(map)); builder.addRepeatedField(field, value); } + } else if (set) { + if (val instanceof IPersistentMap) { + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Map.Entry e = (Map.Entry) s.first(); + Object[] map = {k_item, e.getKey(), k_exists, e.getValue()}; + Object value = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, value); + } + } else { + for (ISeq s = RT.seq(val); s != null; s = s.next()) { + Object[] map = {k_item, s.first(), k_exists, true}; + Object value = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, value); + } + } } else { Object value = toProtoValue(field, val); builder.addRepeatedField(field, value); @@ -463,7 +503,7 @@ public Obj withMeta(IPersistentMap meta) { if(meta != meta()) return new Seq(meta, map, fields, i); return this; } - + public Object first() { if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 77d93af..8bf27ea 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -47,8 +47,8 @@ (is (= ["check" "it" "out"] (p "tags")))) )) (testing "append" - (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"]) - q (protobuf Foo :id 43 :tags ["savory"]) + (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) + q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) r (protobuf Foo :label "bad") s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] @@ -58,6 +58,8 @@ (is (= "bad" (t :label))) (is (= ["sweet"] (t :tags))) (is (= ["sweet" "savory"] (s :tags))) + (is (= #{"foo" "bar" "baz"} (p :tag-set))) + (is (= #{"bap" "baz"} (s :tag-set))) )) (testing "protofields" (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, @@ -78,7 +80,7 @@ (deftest protobuf-extended (testing "create" - (let [p (protobuf Foo :id 5 :tag-set ["little" "yellow"] :attr-map {"size" "little", "color" "yellow", "style" "different"})] + (let [p (protobuf Foo :id 5 :tag-set #{"little" "yellow"} :attr-map {"size" "little", "color" "yellow", "style" "different"})] (is (= #{"little" "yellow"} (:tag-set p))) (is (associative? (:attr-map p))) (is (= "different" (get-in p [:attr-map "style"]))) From f85e20e5d2b117ca93e33ddc95c0b3b5644f3925 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 19 Aug 2010 08:42:12 -0700 Subject: [PATCH 078/525] update to clojure 1.2.0 --- project.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index cf708ff..d9d027e 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,5 @@ -(defproject clojure-protobuf "0.2.7" +(defproject clojure-protobuf "0.2.8" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[clojure "1.2.0-master-SNAPSHOT"] - [clojure-contrib "1.2.0-SNAPSHOT"]] + :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] :jar-files ["proto"]) \ No newline at end of file From 0a203432f14e7b7e79397272154325d9fec598fb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 19 Aug 2010 14:52:00 -0700 Subject: [PATCH 079/525] apply nested message fixes from Ilya Chernikov. thanks! --- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 85822ba..e852d08 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -324,7 +324,7 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v } protected void addField(DynamicMessage.Builder builder, Object key, Object val) { - if (key == null) return; + if (key == null || val == null) return; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); boolean set = field.getOptions().getExtension(Extensions.set); @@ -373,7 +373,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) } else { Object value = toProtoValue(field, val); if (value instanceof DynamicMessage) { - value = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) value); + value = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) value).build(); } builder.setField(field, value); } From 74e659bc4f78f828c460841fd6081e4edbc12f93 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 19 Aug 2010 22:33:33 -0700 Subject: [PATCH 080/525] cake tasks updates. add deps dependency. download google source to lib/ instead of build/ --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index d9d027e..928fd07 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.8" +(defproject clojure-protobuf "0.2.9" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 645fbba..d263b66 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -6,8 +6,8 @@ (:import [org.apache.tools.ant.taskdefs Chmod Copy Delete ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") -(def srcdir (format "build/protobuf-%s" version)) -(def tarfile (format "build/protobuf-%s.tar.gz" version)) +(def srcdir (format "lib/protobuf-%s" version)) +(def tarfile (format "lib/protobuf-%s.tar.gz" version)) (def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) (defn installed? [] @@ -17,7 +17,7 @@ (deftask fetch-protoc (when-not (.exists (file srcdir)) (ant Get {:src url :dest tarfile}) - (ant Untar {:src tarfile :dest "build" :compression "gzip"}))) + (ant Untar {:src tarfile :dest "lib" :compression "gzip"}))) (deftask install-protoc (when-not (installed?) @@ -55,7 +55,7 @@ (defn protoc ([protos] (protoc protos "build/protosrc")) ([protos dest] - (when (or (:recompile *opts*) + (when (or (:force *opts*) (> (modtime "proto") (modtime dest)) (> (modtime "proto") (modtime "classes"))) (ant Mkdir {:dir dest}) @@ -88,7 +88,7 @@ (.substring (.getPath file) (inc (count (.getPath dir)))))) (deftask compile #{proto}) -(deftask proto #{install-protoc} +(deftask proto #{deps install-protoc} "Compile protocol buffer files located in proto dir." (if (= "clojure-protobuf" (:artifact-id *project*)) (do (run-task 'fetch-protoc) From 84bd59134a56ec89559a700d4a42a5600323c11a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 24 Aug 2010 13:28:24 -0700 Subject: [PATCH 081/525] nested message tests from Ilya Chernikov --- proto/clojure/protobuf/test.proto | 10 ++++++++++ test/protobuf_test.clj | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index dd337ef..68b3424 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -36,3 +36,13 @@ message Item { required string item = 1; required bool exists = 2 [default = true]; } + +message ErrorMsg { + required sint32 code = 1; + optional string data = 2; +} + +message Response { + required bool ok = 1; + optional ErrorMsg error = 2; +} \ No newline at end of file diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 8bf27ea..d2baf18 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -3,6 +3,8 @@ (:use clojure.test)) (defprotobuf Foo clojure.protobuf.Test Foo) +(defprotobuf Response clojure.protobuf.Test Response) +(defprotobuf ErrorMsg clojure.protobuf.Test ErrorMsg) (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) @@ -102,3 +104,8 @@ (is (= ["even" "even"] (get-in p [:foo-by-id 6 :tags])))) ))) +(deftest protobuf-nested-message + (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) + +(deftest protobuf-nil-field + (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))])) \ No newline at end of file From cc08c7d93b59b1fe061bc090da98e04c4a5fb93c Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 30 Aug 2010 11:31:51 -0700 Subject: [PATCH 082/525] fix install task to work over socket --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 928fd07..f5619df 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.9" +(defproject clojure-protobuf "0.2.10" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index d263b66..d9603fc 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -20,6 +20,7 @@ (ant Untar {:src tarfile :dest "lib" :compression "gzip"}))) (deftask install-protoc + "Compile and install protoc to /usr/local." (when-not (installed?) (run-task 'fetch-protoc) (when-not (.exists (file srcdir "src" "protoc")) @@ -27,8 +28,16 @@ (ant Chmod {:file (file srcdir "install-sh") :perm "+x"}) (ant ExecTask {:dir srcdir :executable "./configure"}) (ant ExecTask {:dir srcdir :executable "make"})) - (ant ExecTask {:dir srcdir :executable "sudo"} - (args ["make" "install"])))) + (let [password (prompt-read "Password" :echo false)] + (ant ExecTask {:dir srcdir :executable "sudo" :input-string (str password "\n")} + (args ["-S" "make" "install"]))))) + +(deftask uninstall-protoc + "Remove protoc if it is installed." + (when (installed?) + (let [password (prompt-read "Password" :echo false)] + (ant ExecTask {:dir srcdir :executable "sudo" :input-string (str password "\n")} + (args ["-S" "make" "uninstall"]))))) (defn- proto-dependencies "look for lines starting with import in proto-file" [proto-file] From 0f282345c8da7bd363b992c85df9c26e7801f095 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 20 Sep 2010 14:38:45 -0700 Subject: [PATCH 083/525] fix cake tasks --- src/clj/protobuf/tasks.clj | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index d9603fc..8fe199b 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,8 +1,9 @@ (ns protobuf.tasks (:use cake cake.core cake.ant + [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] - [useful.io :only [extract-resource]]) + [cake.utils.io :only [extract-resource]]) (:import [org.apache.tools.ant.taskdefs Chmod Copy Delete ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") @@ -77,9 +78,7 @@ (catch org.apache.tools.ant.BuildException e (ant Delete {:dir dest}) (throw (Exception. (str "error compiling " proto)))))) - (ant Javac {:srcdir (path dest) - :destdir (file "classes") - :classpath (classpath)})))) + (compile-java (file dest))))) (defn build-protobuf [] (ant Mkdir {:dir "proto/google/protobuf"}) From 1c6a768a99c46a444e9168575c8fef93ebad1dd6 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 20 Sep 2010 15:22:41 -0700 Subject: [PATCH 084/525] change conj to behave like clojure maps. for appending behavior use new append method --- src/clj/protobuf.clj | 5 +++- .../protobuf/PersistentProtocolBufferMap.java | 15 ++++++++--- test/protobuf_test.clj | 25 +++++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index fbd0f66..8e3676c 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -42,4 +42,7 @@ (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data length)))) (defn protobuf-dump [#^clojure.protobuf.PersistentProtocolBufferMap p] - (.toByteArray p)) \ No newline at end of file + (.toByteArray p)) + +(defn append [#^clojure.protobuf.PersistentProtocolBufferMap p map] + (.append p map)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index e852d08..49988a5 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -329,6 +329,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { + builder.clearField(field); if (val instanceof Sequential && !set) { for (ISeq s = RT.seq(val); s != null; s = s.next()) { Object value = toProtoValue(field, s.first()); @@ -419,10 +420,7 @@ public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return this; - if (field.isRepeated()) builder.clearField(field); addField(builder, field, val); - return new PersistentProtocolBufferMap(meta(), def, builder); } @@ -433,7 +431,6 @@ public IPersistentMap assocEx(Object key, Object val) throws Exception { public IPersistentCollection cons(Object o) { DynamicMessage.Builder builder = builder(); - if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry) o; addField(builder, e.getKey(), e.getValue()); @@ -450,6 +447,16 @@ public IPersistentCollection cons(Object o) { return new PersistentProtocolBufferMap(meta(), def, builder); } + public PersistentProtocolBufferMap append(IPersistentMap map) { + PersistentProtocolBufferMap proto; + if (map instanceof PersistentProtocolBufferMap) { + proto = (PersistentProtocolBufferMap) map; + } else { + proto = construct(def, map); + } + return new PersistentProtocolBufferMap(meta(), def, builder().mergeFrom(proto.message())); + } + public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return this; diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index d2baf18..ba01280 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -17,13 +17,25 @@ (is (= "bar" (:label p))) (is (= ["little" "yellow"] (:tags p)))) (let [p (conj p {:tags ["different"]})] + (is (= ["different"] (:tags p)))) + (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] + (is (= ["little" "yellow" "different"] (:tags p))) + (is (= "very" (:label p)))) + )) + (testing "append" + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (append p {:label "bar"})] + (is (= 5 (:id p))) + (is (= "bar" (:label p))) + (is (= ["little" "yellow"] (:tags p)))) + (let [p (append p {:tags ["different"]})] (is (= ["little" "yellow" "different"] (:tags p)))) - (let [p (conj p {:tags ["different"] :label "very"})] + (let [p (append p {:tags ["different"] :label "very"})] (is (= ["little" "yellow" "different"] (:tags p))) (is (= "very" (:label p)))) )) (testing "assoc" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] (let [p (assoc p :label "baz" :tags ["nuprin"])] (is (= ["nuprin"] (:tags p))) (is (= "baz" (:label p)))) @@ -31,6 +43,8 @@ (is (= [:yes :no :maybe :no :yes] (:responses p)))) (let [p (assoc p :tags "aspirin")] (is (= ["aspirin"] (:tags p)))) + (let [p (assoc p :foo-by-id {3 {:label "three"} 2 {:label "two"}})] + (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))) )) (testing "dissoc" (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"])] @@ -48,7 +62,7 @@ (is (= ["check" "it" "out"] (p :tags))) (is (= ["check" "it" "out"] (p "tags")))) )) - (testing "append" + (testing "append bytes" (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) r (protobuf Foo :label "bad") @@ -99,9 +113,10 @@ )) (testing "conj" (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] - (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["even"]}}})] + (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) - (is (= ["even" "even"] (get-in p [:foo-by-id 6 :tags])))) + (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) + (is (= "" (get-in p [:foo-by-id 6 :label])))) ))) (deftest protobuf-nested-message From 3863bf1c4de627290969cd159db994c81e64f8d3 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 20 Sep 2010 15:23:30 -0700 Subject: [PATCH 085/525] version 0.2.11-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f5619df..2118252 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.10" +(defproject clojure-protobuf "0.2.11-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] From 5c22eed0ee6c2e85985012b625833f8b843f7d0d Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 4 Oct 2010 12:05:00 -0700 Subject: [PATCH 086/525] add explicit dependency on useful --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2118252..344f93e 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,6 @@ (defproject clojure-protobuf "0.2.11-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[clojure "1.2.0"]] + :dependencies [[clojure "1.2.0"] + [clojure-useful "0.3.0-SNAPSHOT"]] :tasks [protobuf.tasks] :jar-files ["proto"]) \ No newline at end of file From a9ee5bcef6792f762687d719f5e1c6fdf02ce4df Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 4 Oct 2010 21:57:58 -0700 Subject: [PATCH 087/525] make addField robust to missing fields --- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 49988a5..769d190 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -326,6 +326,7 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v protected void addField(DynamicMessage.Builder builder, Object key, Object val) { if (key == null || val == null) return; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + if (field == null) return; boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { From 3731eaac219eade772b809cf20a012c6a8424248 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 6 Oct 2010 14:52:30 -0700 Subject: [PATCH 088/525] add test for double and float. fix cake tasks --- proto/clojure/protobuf/test.proto | 3 +++ src/clj/protobuf/tasks.clj | 2 ++ .../protobuf/PersistentProtocolBufferMap.java | 3 +++ test/protobuf_test.clj | 16 +++++++++++----- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 68b3424..b57e4e8 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -10,6 +10,9 @@ message Foo { optional Foo parent = 4; repeated Response responses = 5; + repeated double doubles = 6; + repeated float floats = 7; + enum Response { YES = 0; NO = 1; diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 8fe199b..21472d1 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,5 +1,7 @@ (ns protobuf.tasks (:use cake cake.core cake.ant + [cake.file :only [file]] + [cake.project :only [log]] [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 769d190..9847a66 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -299,6 +299,9 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v if (value instanceof Integer) return value; Long l = (Long) value; return new Integer(l.intValue()); + case FLOAT: + if (value instanceof Float) return value; + return new Float((Double)value); case ENUM: String name = (value instanceof Keyword) ? ((Keyword) value).getName() : (String) value; name = name.toUpperCase().replaceAll("-","_"); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index ba01280..7a7ec93 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -11,11 +11,14 @@ (deftest protobuf-simple (testing "conj" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] (let [p (conj p {:label "bar"})] (is (= 5 (:id p))) (is (= "bar" (:label p))) - (is (= ["little" "yellow"] (:tags p)))) + (is (= ["little" "yellow"] (:tags p))) + (is (= [1.2 3.4 5.6] (:doubles p))) + (is (= [(float 0.01) (float 0.02) (float 0.03)] (:floats p))) + ) (let [p (conj p {:tags ["different"]})] (is (= ["different"] (:tags p)))) (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] @@ -23,7 +26,7 @@ (is (= "very" (:label p)))) )) (testing "append" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"])] + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] (let [p (append p {:label "bar"})] (is (= 5 (:id p))) (is (= "bar" (:label p))) @@ -33,6 +36,9 @@ (let [p (append p {:tags ["different"] :label "very"})] (is (= ["little" "yellow" "different"] (:tags p))) (is (= "very" (:label p)))) + (let [p (append p {:doubles [3.4] :floats [0.02]})] + (is (= [1.2 3.4] (:doubles p))) + (is (= [(float 0.01) (float 0.02)] (:floats p)))) )) (testing "assoc" (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] @@ -79,7 +85,7 @@ )) (testing "protofields" (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, - :tag_set nil, :attr_map nil, :foo_by_id nil, :groups nil}] + :tag_set nil, :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) (testing "protodefault" @@ -123,4 +129,4 @@ (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) (deftest protobuf-nil-field - (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))])) \ No newline at end of file + (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))])) From 9f657176f67526614f14b3edd105758e862a7601 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 13 Oct 2010 20:12:06 -0700 Subject: [PATCH 089/525] support installing protoc from cake on linux when tty required for sudo --- src/clj/protobuf/tasks.clj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 21472d1..4cbd956 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -2,11 +2,12 @@ (:use cake cake.core cake.ant [cake.file :only [file]] [cake.project :only [log]] + [cake.tasks.deps :only [os-name]] [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] [cake.utils.io :only [extract-resource]]) - (:import [org.apache.tools.ant.taskdefs Chmod Copy Delete ExecTask Get Javac Mkdir Untar])) + (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") (def srcdir (format "lib/protobuf-%s" version)) @@ -31,9 +32,13 @@ (ant Chmod {:file (file srcdir "install-sh") :perm "+x"}) (ant ExecTask {:dir srcdir :executable "./configure"}) (ant ExecTask {:dir srcdir :executable "make"})) - (let [password (prompt-read "Password" :echo false)] - (ant ExecTask {:dir srcdir :executable "sudo" :input-string (str password "\n")} - (args ["-S" "make" "install"]))))) + (let [password (prompt-read "Password" :echo false) + opts {:dir srcdir :input-string (str password "\n")}] + (if (= "linux "(os-name)) + (ant ExecTask (assoc opts :executable "script") + (argline "-q -c 'sudo -S make install' /dev/null")) + (ant ExecTask (assoc opts :executable "sudo") + (args ["-S" "make" "install"])))))) (deftask uninstall-protoc "Remove protoc if it is installed." @@ -78,7 +83,6 @@ (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) (catch org.apache.tools.ant.BuildException e - (ant Delete {:dir dest}) (throw (Exception. (str "error compiling " proto)))))) (compile-java (file dest))))) From a2dbe1e775447f4aae4bf3d63603dc49fffd4f92 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 13 Oct 2010 20:14:01 -0700 Subject: [PATCH 090/525] version 0.2.11 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 344f93e..f763970 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.11-SNAPSHOT" +(defproject clojure-protobuf "0.2.11" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [clojure-useful "0.3.0-SNAPSHOT"]] From d5e149a1357444043a66b69d7b41b046b605ba6a Mon Sep 17 00:00:00 2001 From: ninjudd Date: Sun, 17 Oct 2010 22:47:47 -0700 Subject: [PATCH 091/525] add test for map-by with required field --- proto/clojure/protobuf/test.proto | 9 +++--- test/protobuf_test.clj | 46 ++++++++++++++++++------------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index b57e4e8..9768209 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -19,10 +19,11 @@ message Foo { MAYBE = 1; } - repeated Item tag_set = 10 [(set) = true]; - repeated Pair attr_map = 11 [(map) = true]; - repeated Foo foo_by_id = 12 [(map_by) = "id"]; - repeated Group groups = 13 [(map) = true]; + repeated Item tag_set = 10 [(set) = true]; + repeated Pair attr_map = 11 [(map) = true]; + repeated Foo foo_by_id = 12 [(map_by) = "id"]; + repeated Group groups = 13 [(map) = true]; + repeated Item item_map = 14 [(map_by) = "item"]; } message Pair { diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 7a7ec93..d53ac97 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -82,26 +82,10 @@ (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) (is (= #{"bap" "baz"} (s :tag-set))) - )) - (testing "protofields" - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, - :tag_set nil, :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil}] - (is (= fields (protofields Foo))) - (is (= fields (protofields clojure.protobuf.Test$Foo))))) - (testing "protodefault" - (is (= 43 (protodefault Foo :id))) - (is (= "" (protodefault Foo :label))) - (is (= [] (protodefault Foo :tags))) - (is (= nil (protodefault Foo :parent))) - (is (= [] (protodefault Foo :responses))) - (is (= #{} (protodefault Foo :tag_set))) - (is (= {} (protodefault Foo :foo_by_id))) - (is (= {} (protodefault Foo :groups))) - (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))) - )) + ))) (deftest protobuf-extended - (testing "create" + (comment testing "create" (let [p (protobuf Foo :id 5 :tag-set #{"little" "yellow"} :attr-map {"size" "little", "color" "yellow", "style" "different"})] (is (= #{"little" "yellow"} (:tag-set p))) (is (associative? (:attr-map p))) @@ -117,7 +101,12 @@ (is (= 6 (six :id))) (is (= "six" (six :label)))) )) - (testing "conj" + (testing "map-by with required field" + (let [p (protobuf Foo :id 1 :item-map {"foo" {:exists true} "bar" {:exists false}})] + (is (= "foo" (get-in p [:item-map "foo" :item]))) + (is (= "bar" (get-in p [:item-map "bar" :item]))) + )) + (comment testing "conj" (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) @@ -125,6 +114,25 @@ (is (= "" (get-in p [:foo-by-id 6 :label])))) ))) +(deftest protofields-and-defaults + (testing "protofields" + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, + :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil, :item_map nil}] + (is (= fields (protofields Foo))) + (is (= fields (protofields clojure.protobuf.Test$Foo))))) + (testing "protodefault" + (is (= 43 (protodefault Foo :id))) + (is (= "" (protodefault Foo :label))) + (is (= [] (protodefault Foo :tags))) + (is (= nil (protodefault Foo :parent))) + (is (= [] (protodefault Foo :responses))) + (is (= #{} (protodefault Foo :tag-set))) + (is (= {} (protodefault Foo :foo-by-id))) + (is (= {} (protodefault Foo :groups))) + (is (= {} (protodefault Foo :item-map))) + (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))) + )) + (deftest protobuf-nested-message (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) From b07eb5afe2a0cc5a6dd0f139708a67562328a8d8 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 19 Oct 2010 20:09:43 -0700 Subject: [PATCH 092/525] add offset to protobuf-load function --- project.clj | 2 +- src/clj/protobuf.clj | 4 ++-- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index f763970..727a664 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.2.11" +(defproject clojure-protobuf "0.3.0-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [clojure-useful "0.3.0-SNAPSHOT"]] diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 8e3676c..acc0a87 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -38,8 +38,8 @@ (defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data))) - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data #^Integer length] - (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data length)))) + ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data #^Integer offset #^Integer length] + (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data offset length)))) (defn protobuf-dump [#^clojure.protobuf.PersistentProtocolBufferMap p] (.toByteArray p)) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 9847a66..0f9c46e 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -118,8 +118,8 @@ static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws I return new PersistentProtocolBufferMap(null, def, message); } - static public PersistentProtocolBufferMap create(Def def, byte[] bytes, Integer length) throws IOException { - CodedInputStream input = CodedInputStream.newInstance(bytes, 0, length.intValue()); + static public PersistentProtocolBufferMap create(Def def, byte[] bytes, Integer offset, Integer length) throws IOException { + CodedInputStream input = CodedInputStream.newInstance(bytes, offset.intValue(), length.intValue()); DynamicMessage message = def.parseFrom(input); return new PersistentProtocolBufferMap(null, def, message); } From 13a75beb0068fc71cb4489d32ce504dd60bddb01 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 20 Oct 2010 11:49:25 -0700 Subject: [PATCH 093/525] replace _ with - in protofields --- src/clj/protobuf.clj | 5 +++-- test/protobuf_test.clj | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index acc0a87..3997852 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -32,8 +32,9 @@ type #^com.google.protobuf.Descriptors$Descriptor (.getMessageType type)] (into {} (for [#^com.google.protobuf.Descriptors$FieldDescriptor field (.getFields type)] - (let [meta-string (.. field getOptions (getExtension (clojure.protobuf.Extensions/meta)))] - [(keyword (.getName field)) (when-not (empty? meta-string) (read-string meta-string))]))))) + (let [meta-string (.. field getOptions (getExtension (clojure.protobuf.Extensions/meta))) + field-name (keyword (.replaceAll (.getName field) "_" "-"))] + [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index d53ac97..96a655a 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -116,8 +116,8 @@ (deftest protofields-and-defaults (testing "protofields" - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, - :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil, :item_map nil}] + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, + :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) (testing "protodefault" From d9505d09f1af3b03509fcd3e675a01fa49b294aa Mon Sep 17 00:00:00 2001 From: ninjudd Date: Thu, 4 Nov 2010 17:09:54 -0700 Subject: [PATCH 094/525] fix typo --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 4cbd956..3ba5c06 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -34,7 +34,7 @@ (ant ExecTask {:dir srcdir :executable "make"})) (let [password (prompt-read "Password" :echo false) opts {:dir srcdir :input-string (str password "\n")}] - (if (= "linux "(os-name)) + (if (= "linux" (os-name)) (ant ExecTask (assoc opts :executable "script") (argline "-q -c 'sudo -S make install' /dev/null")) (ant ExecTask (assoc opts :executable "sudo") From c15e26c7b15be4d8f989616ec093b741c3c60a6d Mon Sep 17 00:00:00 2001 From: ninjudd Date: Thu, 18 Nov 2010 17:47:13 -0800 Subject: [PATCH 095/525] return nil when non-repeated field is missing. clean up tests --- .../protobuf/PersistentProtocolBufferMap.java | 1 + test/protobuf_test.clj | 52 +++++++++---------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 0f9c46e..e70cf56 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -412,6 +412,7 @@ public IMapEntry entryAt(Object key) { public Object valAt(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return null; + if (!field.isRepeated() && !message().hasField(field)) return null; return fromProtoValue(field, message().getField(field)); } diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 96a655a..bc5fc75 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -10,6 +10,7 @@ (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) (deftest protobuf-simple + (testing "conj" (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] (let [p (conj p {:label "bar"})] @@ -23,8 +24,8 @@ (is (= ["different"] (:tags p)))) (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] (is (= ["little" "yellow" "different"] (:tags p))) - (is (= "very" (:label p)))) - )) + (is (= "very" (:label p)))))) + (testing "append" (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] (let [p (append p {:label "bar"})] @@ -38,8 +39,8 @@ (is (= "very" (:label p)))) (let [p (append p {:doubles [3.4] :floats [0.02]})] (is (= [1.2 3.4] (:doubles p))) - (is (= [(float 0.01) (float 0.02)] (:floats p)))) - )) + (is (= [(float 0.01) (float 0.02)] (:floats p)))))) + (testing "assoc" (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] (let [p (assoc p :label "baz" :tags ["nuprin"])] @@ -50,14 +51,14 @@ (let [p (assoc p :tags "aspirin")] (is (= ["aspirin"] (:tags p)))) (let [p (assoc p :foo-by-id {3 {:label "three"} 2 {:label "two"}})] - (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))) - )) + (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))))) + (testing "dissoc" - (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"])] + (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] (let [p (dissoc p :label :tags)] - (is (= [] (:tags p))) - (is (= "" (:label p)))) - )) + (is (= [] (:tags p))) + (is (= nil (:label p)))))) + (testing "string keys" (let [p (protobuf Foo "id" 5 "label" "rad")] (is (= 5 (p :id))) @@ -66,8 +67,8 @@ (is (= "rad" (p "label"))) (let [p (conj p {"tags" ["check" "it" "out"]})] (is (= ["check" "it" "out"] (p :tags))) - (is (= ["check" "it" "out"] (p "tags")))) - )) + (is (= ["check" "it" "out"] (p "tags")))))) + (testing "append bytes" (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) @@ -81,45 +82,45 @@ (is (= ["sweet"] (t :tags))) (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) - (is (= #{"bap" "baz"} (s :tag-set))) - ))) + (is (= #{"bap" "baz"} (s :tag-set)))))) (deftest protobuf-extended - (comment testing "create" + + (testing "create" (let [p (protobuf Foo :id 5 :tag-set #{"little" "yellow"} :attr-map {"size" "little", "color" "yellow", "style" "different"})] (is (= #{"little" "yellow"} (:tag-set p))) (is (associative? (:attr-map p))) (is (= "different" (get-in p [:attr-map "style"]))) (is (= "little" (get-in p [:attr-map "size" ]))) - (is (= "yellow" (get-in p [:attr-map "color"]))) - ) + (is (= "yellow" (get-in p [:attr-map "color"])))) (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five"}, 6 {:label "six"}})] (let [five ((p :foo-by-id) 5) six ((p :foo-by-id) 6)] (is (= 5 (five :id))) (is (= "five" (five :label))) (is (= 6 (six :id))) - (is (= "six" (six :label)))) - )) + (is (= "six" (six :label)))))) + (testing "map-by with required field" (let [p (protobuf Foo :id 1 :item-map {"foo" {:exists true} "bar" {:exists false}})] (is (= "foo" (get-in p [:item-map "foo" :item]))) - (is (= "bar" (get-in p [:item-map "bar" :item]))) - )) - (comment testing "conj" + (is (= "bar" (get-in p [:item-map "bar" :item]))))) + + (testing "conj" (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) - (is (= "" (get-in p [:foo-by-id 6 :label])))) - ))) + (is (= nil (get-in p [:foo-by-id 6 :label]))))))) (deftest protofields-and-defaults + (testing "protofields" (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) + (testing "protodefault" (is (= 43 (protodefault Foo :id))) (is (= "" (protodefault Foo :label))) @@ -130,8 +131,7 @@ (is (= {} (protodefault Foo :foo-by-id))) (is (= {} (protodefault Foo :groups))) (is (= {} (protodefault Foo :item-map))) - (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))) - )) + (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))))) (deftest protobuf-nested-message (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) From ac58b619166a5a160eb801940caaa7961883d571 Mon Sep 17 00:00:00 2001 From: lance bradley Date: Wed, 17 Nov 2010 15:34:55 -0800 Subject: [PATCH 096/525] cake face compatible branch --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 3ba5c06..ce6b8de 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -2,7 +2,7 @@ (:use cake cake.core cake.ant [cake.file :only [file]] [cake.project :only [log]] - [cake.tasks.deps :only [os-name]] + [cake.utils :only [os-name prompt-read]] [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] From 2b222bd5597819726a10a3a9773eb396e1cdda24 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 22 Nov 2010 17:04:31 -0800 Subject: [PATCH 097/525] version 0.3.1-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 727a664..b74b63e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.0-SNAPSHOT" +(defproject clojure-protobuf "0.3.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [clojure-useful "0.3.0-SNAPSHOT"]] From 4ae56cd3868c65420f97bdd7fce9350ede653712 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 22 Nov 2010 17:18:07 -0800 Subject: [PATCH 098/525] update for cake face branch --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index ce6b8de..418ba09 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,7 +1,7 @@ (ns protobuf.tasks (:use cake cake.core cake.ant [cake.file :only [file]] - [cake.project :only [log]] + [bake.core :only [log]] [cake.utils :only [os-name prompt-read]] [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] From c591ee0b4b9dcef17e9414ed957b41186cd46093 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 30 Nov 2010 17:13:46 -0800 Subject: [PATCH 099/525] cake 0.6.0 fixes --- src/clj/protobuf/tasks.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 418ba09..052f257 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -6,7 +6,7 @@ [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] - [cake.utils.io :only [extract-resource]]) + [cake.utils :only [extract-resource]]) (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") @@ -26,7 +26,7 @@ (deftask install-protoc "Compile and install protoc to /usr/local." (when-not (installed?) - (run-task 'fetch-protoc) + (invoke fetch-protoc) (when-not (.exists (file srcdir "src" "protoc")) (ant Chmod {:file (file srcdir "configure") :perm "+x"}) (ant Chmod {:file (file srcdir "install-sh") :perm "+x"}) @@ -105,6 +105,6 @@ (deftask proto #{deps install-protoc} "Compile protocol buffer files located in proto dir." (if (= "clojure-protobuf" (:artifact-id *project*)) - (do (run-task 'fetch-protoc) + (do (invoke fetch-protoc) (build-protobuf)) (protoc (or (:proto *opts*) (proto-files (file "proto")))))) From 731e2df99072f0c8ba1dddc13e29e0cae11a4ee2 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Thu, 9 Dec 2010 12:34:36 -0800 Subject: [PATCH 100/525] move extract-resource to bake.io --- project.clj | 3 +-- src/clj/protobuf/tasks.clj | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index b74b63e..7bc938c 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,5 @@ (defproject clojure-protobuf "0.3.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[clojure "1.2.0"] - [clojure-useful "0.3.0-SNAPSHOT"]] + :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] :jar-files ["proto"]) \ No newline at end of file diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 052f257..11f07bc 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -6,7 +6,7 @@ [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] - [cake.utils :only [extract-resource]]) + [bake.io :only [extract-resource]]) (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) (def version "2.3.0") From 4da20d6ccf9817f297dd7d0f3f5d8c58a81440ac Mon Sep 17 00:00:00 2001 From: ninjudd Date: Sun, 12 Dec 2010 13:52:09 -0800 Subject: [PATCH 101/525] switch to use uncle --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 11f07bc..94e1fbf 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,5 +1,5 @@ (ns protobuf.tasks - (:use cake cake.core cake.ant + (:use cake cake.core uncle.core [cake.file :only [file]] [bake.core :only [log]] [cake.utils :only [os-name prompt-read]] From 0afddcd76a5a56c92cb5762a4f79b554991b3f97 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 29 Dec 2010 13:11:21 -0800 Subject: [PATCH 102/525] remove version number from README --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2989334..8e262ca 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,11 @@ Then you can access the maps in clojure: ## Installation -Add the following to your [Cake](http://github.com/ninjudd/cake) project.clj: +Add the `clojure-protobuf` to your [Cake](http://github.com/ninjudd/cake) project.clj: - :dependencies [[clojure-protobuf "0.2.4"]] - :dev-dependencies [[clojure-protobuf "0.2.4"]] + :dependencies [[clojure-protobuf "LATEST"]] + :dev-dependencies [[clojure-protobuf "LATEST"]] :tasks [protobuf.tasks] -This code works with Clojure 1.2. Here's the [1.1 branch](http://github.com/ninjudd/clojure-protobuf/tree/clojure-1.1). \ No newline at end of file +You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget +to update this every time I bump the version, so I'm not going to put an actual number here. \ No newline at end of file From afcc6629027eb57926f797784d6f3d1aa60f1d6a Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 3 Jan 2011 17:56:23 -0800 Subject: [PATCH 103/525] coerce integers to floats or doubles --- proto/clojure/protobuf/test.proto | 3 +++ .../protobuf/PersistentProtocolBufferMap.java | 9 +++++++-- test/protobuf_test.clj | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 9768209..3b89fb1 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -13,6 +13,9 @@ message Foo { repeated double doubles = 6; repeated float floats = 7; + optional double lat = 8; + optional float long = 9; + enum Response { YES = 0; NO = 1; diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index e70cf56..8753145 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -300,8 +300,13 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v Long l = (Long) value; return new Integer(l.intValue()); case FLOAT: - if (value instanceof Float) return value; - return new Float((Double)value); + if (value instanceof Integer) return new Float((Integer) value * 1.0); + if (value instanceof Double) return new Float((Double) value); + return value; + case DOUBLE: + if (value instanceof Integer) return new Double((Integer) value * 1.0); + if (value instanceof Float) return new Double((Float) value); + return value; case ENUM: String name = (value instanceof Keyword) ? ((Keyword) value).getName() : (String) value; name = name.toUpperCase().replaceAll("-","_"); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index bc5fc75..07509df 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -82,7 +82,16 @@ (is (= ["sweet"] (t :tags))) (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) - (is (= #{"bap" "baz"} (s :tag-set)))))) + (is (= #{"bap" "baz"} (s :tag-set))))) + + (testing "coercing doubles and floats" + (let [p (protobuf Foo :lat 5 :long 6)] + (is (= 5.0 (p :lat))) + (is (= 6.0 (p :long)))) + (let [p (protobuf Foo :lat (float 5.0) :long (double 6.0))] + (is (= 5.0 (p :lat))) + (is (= 6.0 (p :long)))) + )) (deftest protobuf-extended @@ -117,12 +126,14 @@ (testing "protofields" (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, - :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil}] + :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil, :lat nil, :long nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) (testing "protodefault" (is (= 43 (protodefault Foo :id))) + (is (= 0.0 (protodefault Foo :lat))) + (is (= 0.0 (protodefault Foo :long))) (is (= "" (protodefault Foo :label))) (is (= [] (protodefault Foo :tags))) (is (= nil (protodefault Foo :parent))) From abbdb952f0787267f1b6cb2e420955aabd7b0150 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 3 Jan 2011 17:56:50 -0800 Subject: [PATCH 104/525] version 0.3.1 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 7bc938c..d9aa063 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ -(defproject clojure-protobuf "0.3.1-SNAPSHOT" +(defproject clojure-protobuf "0.3.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] - :jar-files ["proto"]) \ No newline at end of file + :jar-files ["proto"]) From a3a639b2f089ab4d40e7d869b71153282388a1dd Mon Sep 17 00:00:00 2001 From: David Santiago Date: Thu, 10 Feb 2011 17:07:30 -0600 Subject: [PATCH 105/525] Changed compile-native to work with changes to cake's compile task. Bumped version to 0.3.2 --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index d9aa063..e8deeba 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.1" +(defproject clojure-protobuf "0.3.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 94e1fbf..79d6739 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -101,7 +101,7 @@ (for [file (rest (file-seq dir)) :when (proto-file? file)] (.substring (.getPath file) (inc (count (.getPath dir)))))) -(deftask compile #{proto}) +(deftask compile-native #{proto}) (deftask proto #{deps install-protoc} "Compile protocol buffer files located in proto dir." (if (= "clojure-protobuf" (:artifact-id *project*)) From a12631c2cca14c4f9c9ec88c6959ae867500650e Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 15 Feb 2011 12:56:58 -0800 Subject: [PATCH 106/525] add global option to use underscores (since we must clear all keyword caches when it is changed)... clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores --- project.clj | 2 +- proto/clojure/protobuf/test.proto | 7 +-- src/clj/protobuf.clj | 44 ++++++++++--------- .../protobuf/PersistentProtocolBufferMap.java | 22 ++++++++-- test/protobuf_test.clj | 11 +++++ 5 files changed, 58 insertions(+), 28 deletions(-) diff --git a/project.clj b/project.clj index e8deeba..6f92ba3 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.2" +(defproject clojure-protobuf "0.3.3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 3b89fb1..6781bd9 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -17,9 +17,10 @@ message Foo { optional float long = 9; enum Response { - YES = 0; - NO = 1; - MAYBE = 1; + YES = 0; + NO = 1; + MAYBE = 2; + NOT_SURE = 3; } repeated Item tag_set = 10 [(set) = true]; diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 3997852..d8140bd 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,49 +1,51 @@ -(ns protobuf) +(ns protobuf + (:import (clojure.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) + (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor))) (defn protobuf? [obj] - (instance? clojure.protobuf.PersistentProtocolBufferMap obj)) + (instance? PersistentProtocolBufferMap obj)) (defn protodef? [obj] - (instance? clojure.protobuf.PersistentProtocolBufferMap$Def obj)) + (instance? PersistentProtocolBufferMap$Def obj)) (defn protodef [class] (if (or (protodef? class) (nil? class)) class - (clojure.protobuf.PersistentProtocolBufferMap$Def/create class))) + (PersistentProtocolBufferMap$Def/create class))) (defmacro defprotobuf [sym & args] (let [class (apply str (interpose "$" (map name args)))] `(def ~sym (protodef ~class)))) (defn protobuf - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type] - (clojure.protobuf.PersistentProtocolBufferMap/construct type {})) - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type m] - (clojure.protobuf.PersistentProtocolBufferMap/construct type m)) - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type k v & kvs] - (clojure.protobuf.PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) + ([#^PersistentProtocolBufferMap$Def type] + (PersistentProtocolBufferMap/construct type {})) + ([#^PersistentProtocolBufferMap$Def type m] + (PersistentProtocolBufferMap/construct type m)) + ([#^PersistentProtocolBufferMap$Def type k v & kvs] + (PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) (defn protodefault [type key] - (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type)] + (let [type #^PersistentProtocolBufferMap$Def (protodef type)] (.defaultValue type key))) (defn protofields [type] - (let [type #^clojure.protobuf.PersistentProtocolBufferMap$Def (protodef type) - type #^com.google.protobuf.Descriptors$Descriptor (.getMessageType type)] + (let [type #^PersistentProtocolBufferMap$Def (protodef type) + type #^Descriptors$Descriptor (.getMessageType type)] (into {} - (for [#^com.google.protobuf.Descriptors$FieldDescriptor field (.getFields type)] - (let [meta-string (.. field getOptions (getExtension (clojure.protobuf.Extensions/meta))) + (for [#^Descriptors$FieldDescriptor field (.getFields type)] + (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) field-name (keyword (.replaceAll (.getName field) "_" "-"))] [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data] - (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data))) - ([#^clojure.protobuf.PersistentProtocolBufferMap$Def type #^bytes data #^Integer offset #^Integer length] - (if data (clojure.protobuf.PersistentProtocolBufferMap/create type data offset length)))) + ([#^PersistentProtocolBufferMap$Def type #^bytes data] + (if data (PersistentProtocolBufferMap/create type data))) + ([#^PersistentProtocolBufferMap$Def type #^bytes data #^Integer offset #^Integer length] + (if data (PersistentProtocolBufferMap/create type data offset length)))) -(defn protobuf-dump [#^clojure.protobuf.PersistentProtocolBufferMap p] +(defn protobuf-dump [#^PersistentProtocolBufferMap p] (.toByteArray p)) -(defn append [#^clojure.protobuf.PersistentProtocolBufferMap p map] +(defn append [#^PersistentProtocolBufferMap p map] (.append p map)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 8753145..8e0fe45 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -171,12 +171,28 @@ protected DynamicMessage.Builder builder() { } } + + static boolean use_underscores = false; + static public void setUseUnderscores(boolean val) { + use_underscores = val; + enum_to_keyword.clear(); + map_field_by.clear(); + } + + static protected String normalize(String s) { + s = s.toLowerCase(); + if (!use_underscores) { + s = s.replaceAll("_","-"); + } + return s; + } + static ConcurrentHashMap enum_to_keyword = new ConcurrentHashMap(); static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { Keyword keyword = enum_to_keyword.get(enum_value); if (keyword == null) { - String name = enum_value.getName().toLowerCase().replaceAll("_","-"); + String name = normalize(enum_value.getName()); keyword = Keyword.intern(Symbol.intern(name)); enum_to_keyword.putIfAbsent(enum_value, keyword); } @@ -192,7 +208,7 @@ static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { if (keyword == null) { String name = field.getOptions().getExtension(Extensions.mapBy); - name = name.toLowerCase().replaceAll("_","-"); + name = normalize(name); keyword = Keyword.intern(Symbol.intern(name)); map_field_by.putIfAbsent(field, keyword); } @@ -524,7 +540,7 @@ public Obj withMeta(IPersistentMap meta) { public Object first() { if (i == fields.length) return null; Descriptors.FieldDescriptor field = fields[i]; - String name = field.getName().replaceAll("_","-"); + String name = normalize(field.getName()); Keyword key = Keyword.intern(Symbol.intern(name)); Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); return new MapEntry(key, val); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 07509df..7cea7b0 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -144,6 +144,17 @@ (is (= {} (protodefault Foo :item-map))) (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))))) +(deftest use-underscores + (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] + (is (= '(:responses :tag-set) (keys p))) + (is (= [:yes :not-sure :maybe :not-sure :no] (:responses p))) + + (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores true) + (is (= '(:responses :tag_set) (keys p))) + (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) + + (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) + (deftest protobuf-nested-message (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) From 39c34b179a1caefcc7d5a4e0c2864c1820ba01ee Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 15 Feb 2011 13:39:17 -0800 Subject: [PATCH 107/525] normalize protofields --- src/clj/protobuf.clj | 2 +- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 3 +-- test/protobuf_test.clj | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index d8140bd..00c9572 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -35,7 +35,7 @@ (into {} (for [#^Descriptors$FieldDescriptor field (.getFields type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (keyword (.replaceAll (.getName field) "_" "-"))] + field-name (keyword (PersistentProtocolBufferMap/normalize (.getName field)))] [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 8e0fe45..68e3dac 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -171,7 +171,6 @@ protected DynamicMessage.Builder builder() { } } - static boolean use_underscores = false; static public void setUseUnderscores(boolean val) { use_underscores = val; @@ -179,7 +178,7 @@ static public void setUseUnderscores(boolean val) { map_field_by.clear(); } - static protected String normalize(String s) { + static public String normalize(String s) { s = s.toLowerCase(); if (!use_underscores) { s = s.replaceAll("_","-"); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 7cea7b0..fbb899c 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -153,6 +153,10 @@ (is (= '(:responses :tag_set) (keys p))) (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, + :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil, :item_map nil, :lat nil, :long nil}] + (is (= fields (protofields Foo)))) + (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) (deftest protobuf-nested-message From 14bd6d348cea34cf012fcc21ec4b1d8436f11e72 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 18 Feb 2011 13:59:58 -0800 Subject: [PATCH 108/525] move os-name to bake.core --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index 6f92ba3..5280b3b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.3" +(defproject clojure-protobuf "0.3.4-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 79d6739..08527f7 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -1,8 +1,8 @@ (ns protobuf.tasks (:use cake cake.core uncle.core [cake.file :only [file]] - [bake.core :only [log]] - [cake.utils :only [os-name prompt-read]] + [bake.core :only [log os-name]] + [cake.utils :only [prompt-read]] [cake.tasks.compile :only [compile-java]] [clojure.java.shell :only [sh]] [clojure.java.io :only [reader]] @@ -74,7 +74,7 @@ ([protos dest] (when (or (:force *opts*) (> (modtime "proto") (modtime dest)) - (> (modtime "proto") (modtime "classes"))) + (> (modtime "proto") (modtime "classes"))) (ant Mkdir {:dir dest}) (ant Mkdir {:dir "build/proto"}) (doseq [proto protos] From 4d94e093e742db9e867e9b514ce72a35eb4f36e6 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 21 Feb 2011 09:14:50 -0800 Subject: [PATCH 109/525] fix type hints. add read method that takes a CodedInputStream --- src/clj/protobuf.clj | 33 ++++++++++--------- .../protobuf/PersistentProtocolBufferMap.java | 5 ++- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 00c9572..d75b7c5 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,6 +1,6 @@ (ns protobuf (:import (clojure.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) - (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor))) + (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream))) (defn protobuf? [obj] (instance? PersistentProtocolBufferMap obj)) @@ -18,34 +18,37 @@ `(def ~sym (protodef ~class)))) (defn protobuf - ([#^PersistentProtocolBufferMap$Def type] + ([^PersistentProtocolBufferMap$Def type] (PersistentProtocolBufferMap/construct type {})) - ([#^PersistentProtocolBufferMap$Def type m] + ([^PersistentProtocolBufferMap$Def type m] (PersistentProtocolBufferMap/construct type m)) - ([#^PersistentProtocolBufferMap$Def type k v & kvs] + ([^PersistentProtocolBufferMap$Def type k v & kvs] (PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) (defn protodefault [type key] - (let [type #^PersistentProtocolBufferMap$Def (protodef type)] + (let [type ^PersistentProtocolBufferMap$Def (protodef type)] (.defaultValue type key))) (defn protofields [type] - (let [type #^PersistentProtocolBufferMap$Def (protodef type) - type #^Descriptors$Descriptor (.getMessageType type)] + (let [type ^PersistentProtocolBufferMap$Def (protodef type) + type ^Descriptors$Descriptor (.getMessageType type)] (into {} - (for [#^Descriptors$FieldDescriptor field (.getFields type)] + (for [^Descriptors$FieldDescriptor field (.getFields type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) field-name (keyword (PersistentProtocolBufferMap/normalize (.getName field)))] [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load - ([#^PersistentProtocolBufferMap$Def type #^bytes data] - (if data (PersistentProtocolBufferMap/create type data))) - ([#^PersistentProtocolBufferMap$Def type #^bytes data #^Integer offset #^Integer length] - (if data (PersistentProtocolBufferMap/create type data offset length)))) - -(defn protobuf-dump [#^PersistentProtocolBufferMap p] + ([^PersistentProtocolBufferMap$Def type ^bytes data] + (when data + (PersistentProtocolBufferMap/create type data))) + ([^PersistentProtocolBufferMap$Def type ^bytes data ^Integer offset ^Integer length] + (when data + (let [^CodedInputStream in (CodedInputStream/newInstance data offset length)] + (PersistentProtocolBufferMap/read type in))))) + +(defn protobuf-dump [^PersistentProtocolBufferMap p] (.toByteArray p)) -(defn append [#^PersistentProtocolBufferMap p map] +(defn append [^PersistentProtocolBufferMap p map] (.append p map)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 68e3dac..63b8b3d 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -118,9 +118,8 @@ static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws I return new PersistentProtocolBufferMap(null, def, message); } - static public PersistentProtocolBufferMap create(Def def, byte[] bytes, Integer offset, Integer length) throws IOException { - CodedInputStream input = CodedInputStream.newInstance(bytes, offset.intValue(), length.intValue()); - DynamicMessage message = def.parseFrom(input); + static public PersistentProtocolBufferMap read(Def def, CodedInputStream in) throws IOException { + DynamicMessage message = def.parseFrom(in); return new PersistentProtocolBufferMap(null, def, message); } From e875dcd113de72216a82ad2ce669f39bd6425fcd Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 21 Feb 2011 21:25:10 -0800 Subject: [PATCH 110/525] add support for reading delimited protobufs from input streams and writing delimited protobufs to output streams --- src/clj/protobuf.clj | 21 ++++++++++-- .../protobuf/PersistentProtocolBufferMap.java | 32 +++++++++++++++++-- test/protobuf_test.clj | 14 +++++++- 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index d75b7c5..68d6319 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,6 +1,8 @@ (ns protobuf + (:use [clojure.java.io :only [input-stream output-stream]]) (:import (clojure.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) - (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream))) + (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream) + (java.io InputStream OutputStream))) (defn protobuf? [obj] (instance? PersistentProtocolBufferMap obj)) @@ -45,10 +47,25 @@ ([^PersistentProtocolBufferMap$Def type ^bytes data ^Integer offset ^Integer length] (when data (let [^CodedInputStream in (CodedInputStream/newInstance data offset length)] - (PersistentProtocolBufferMap/read type in))))) + (PersistentProtocolBufferMap/parseFrom type in))))) + (defn protobuf-dump [^PersistentProtocolBufferMap p] (.toByteArray p)) +(defn protobuf-seq [^PersistentProtocolBufferMap$Def type in] + (lazy-seq + (io! + (let [^InputStream in (input-stream in)] + (when-let [p (PersistentProtocolBufferMap/parseDelimitedFrom type in)] + (cons p (protobuf-seq type in))))))) + +(defn protobuf-write [out & ps] + (io! + (let [^OutputStream out (output-stream out)] + (doseq [^PersistentProtocolBufferMap p ps] + (.writeDelimitedTo p out)) + (.flush out)))) + (defn append [^PersistentProtocolBufferMap p map] (.append p map)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 63b8b3d..0e498a4 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -13,6 +13,7 @@ import clojure.lang.*; import java.util.*; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintWriter; import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; @@ -22,6 +23,7 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.DescriptorProtos; import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; public class PersistentProtocolBufferMap extends APersistentMap { public static class Def { @@ -61,6 +63,15 @@ public DynamicMessage parseFrom(CodedInputStream input) throws IOException { return DynamicMessage.parseFrom(type, input); } + public DynamicMessage.Builder parseDelimitedFrom(InputStream input) throws IOException { + DynamicMessage.Builder builder = newBuilder(); + if (builder.mergeDelimitedFrom(input)) { + return builder; + } else { + return null; + } + } + public DynamicMessage.Builder newBuilder() { return DynamicMessage.newBuilder(type); } @@ -118,11 +129,20 @@ static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws I return new PersistentProtocolBufferMap(null, def, message); } - static public PersistentProtocolBufferMap read(Def def, CodedInputStream in) throws IOException { - DynamicMessage message = def.parseFrom(in); + static public PersistentProtocolBufferMap parseFrom(Def def, CodedInputStream input) throws IOException { + DynamicMessage message = def.parseFrom(input); return new PersistentProtocolBufferMap(null, def, message); } + static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStream input) throws IOException { + DynamicMessage.Builder builder = def.parseDelimitedFrom(input); + if (builder != null) { + return new PersistentProtocolBufferMap(null, def, builder); + } else { + return null; + } + } + static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); @@ -150,6 +170,14 @@ public byte[] toByteArray() { return message().toByteArray(); } + public void writeTo(CodedOutputStream output) throws IOException { + message().writeTo(output); + } + + public void writeDelimitedTo(OutputStream output) throws IOException { + message().writeDelimitedTo(output); + } + public Descriptors.Descriptor getMessageType() { return def.getMessageType(); } diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index fbb899c..8d9f34b 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -1,6 +1,7 @@ (ns protobuf-test (:use protobuf) - (:use clojure.test)) + (:use clojure.test) + (:import (java.io PipedInputStream PipedOutputStream))) (defprotobuf Foo clojure.protobuf.Test Foo) (defprotobuf Response clojure.protobuf.Test Response) @@ -164,3 +165,14 @@ (deftest protobuf-nil-field (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))])) + +(deftest protobuf-seq-and-write-protobuf + (let [in (PipedInputStream.) + out (PipedOutputStream. in) + foo (protobuf Foo :id 1 :label "foo") + bar (protobuf Foo :id 2 :label "bar") + baz (protobuf Foo :id 3 :label "baz")] + (protobuf-write out foo bar baz) + (.close out) + (is (= [{:id 1, :label "foo"} {:id 2, :label "bar"} {:id 3, :label "baz"}] + (protobuf-seq Foo in))))) From dab8327623c0291f030bfc391179335dd0afb3b9 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 21 Feb 2011 21:46:38 -0800 Subject: [PATCH 111/525] docstrings --- src/clj/protobuf.clj | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 68d6319..b0142e1 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -4,22 +4,31 @@ (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream) (java.io InputStream OutputStream))) -(defn protobuf? [obj] +(defn protobuf? + "Is the given object a PersistentProtocolBufferMap?" + [obj] (instance? PersistentProtocolBufferMap obj)) -(defn protodef? [obj] +(defn protodef? + "Is the given object a PersistentProtocolBufferMap$Def?" + [obj] (instance? PersistentProtocolBufferMap$Def obj)) -(defn protodef [class] +(defn protodef + "Create a protodef from a string or protobuf class." + [class] (if (or (protodef? class) (nil? class)) class (PersistentProtocolBufferMap$Def/create class))) -(defmacro defprotobuf [sym & args] +(defmacro defprotobuf + "Helper macro for defining a protodef object." + [sym & args] (let [class (apply str (interpose "$" (map name args)))] `(def ~sym (protodef ~class)))) (defn protobuf + "Construct a protobuf of the given type." ([^PersistentProtocolBufferMap$Def type] (PersistentProtocolBufferMap/construct type {})) ([^PersistentProtocolBufferMap$Def type m] @@ -27,11 +36,15 @@ ([^PersistentProtocolBufferMap$Def type k v & kvs] (PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) -(defn protodefault [type key] +(defn protodefault + "Return the default empty protobuf of the given type." + [type key] (let [type ^PersistentProtocolBufferMap$Def (protodef type)] (.defaultValue type key))) -(defn protofields [type] +(defn protofields + "Return a map of the protobuf fields to the clojure.protobuf.Extensions metadata for each field." + [type] (let [type ^PersistentProtocolBufferMap$Def (protodef type) type ^Descriptors$Descriptor (.getMessageType type)] (into {} @@ -41,6 +54,7 @@ [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load + "Load a protobuf of the given type from an array of bytes." ([^PersistentProtocolBufferMap$Def type ^bytes data] (when data (PersistentProtocolBufferMap/create type data))) @@ -50,22 +64,30 @@ (PersistentProtocolBufferMap/parseFrom type in))))) -(defn protobuf-dump [^PersistentProtocolBufferMap p] +(defn protobuf-dump + "Return the byte representation of the given protobuf." + [^PersistentProtocolBufferMap p] (.toByteArray p)) -(defn protobuf-seq [^PersistentProtocolBufferMap$Def type in] +(defn protobuf-seq + "Lazily read a sequence of length-delimited protobufs of the specified type from the given input stream." + [^PersistentProtocolBufferMap$Def type in] (lazy-seq (io! (let [^InputStream in (input-stream in)] (when-let [p (PersistentProtocolBufferMap/parseDelimitedFrom type in)] (cons p (protobuf-seq type in))))))) -(defn protobuf-write [out & ps] +(defn protobuf-write + "Write the given protobufs to the given output stream, prefixing each with its length to delimit them." + [out & ps] (io! (let [^OutputStream out (output-stream out)] (doseq [^PersistentProtocolBufferMap p ps] (.writeDelimitedTo p out)) (.flush out)))) -(defn append [^PersistentProtocolBufferMap p map] +(defn append + "Merge the given map into the protobuf. Equivalent to appending the byte representations." + [^PersistentProtocolBufferMap p map] (.append p map)) \ No newline at end of file From 069a7d08e15dd008961f9171f73e08e5b5f4199d Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 18 Mar 2011 12:19:50 -0700 Subject: [PATCH 112/525] version 0.3.4 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5280b3b..7003d9e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.4-SNAPSHOT" +(defproject clojure-protobuf "0.3.4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] From 2b6ca58cef87d8f0512d5dbc2d1e6e4fa38721d3 Mon Sep 17 00:00:00 2001 From: Jan Rychter Date: Thu, 7 Apr 2011 15:24:01 +0800 Subject: [PATCH 113/525] make protobuf-seq close the stream after entire sequence is consumed --- src/clj/protobuf.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index b0142e1..78e10a8 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -75,8 +75,9 @@ (lazy-seq (io! (let [^InputStream in (input-stream in)] - (when-let [p (PersistentProtocolBufferMap/parseDelimitedFrom type in)] - (cons p (protobuf-seq type in))))))) + (if-let [p (PersistentProtocolBufferMap/parseDelimitedFrom type in)] + (cons p (protobuf-seq type in)) + (.close in)))))) (defn protobuf-write "Write the given protobufs to the given output stream, prefixing each with its length to delimit them." From d9d08eb7bc8b932bcb5569d1b5c69b52317237ad Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 15 Apr 2011 18:34:23 -0700 Subject: [PATCH 114/525] test cleanup --- test/protobuf_test.clj | 283 ++++++++++++++++++++++------------------- 1 file changed, 149 insertions(+), 134 deletions(-) diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 8d9f34b..3e0f4da 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -10,140 +10,155 @@ (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) -(deftest protobuf-simple - - (testing "conj" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] - (let [p (conj p {:label "bar"})] - (is (= 5 (:id p))) - (is (= "bar" (:label p))) - (is (= ["little" "yellow"] (:tags p))) - (is (= [1.2 3.4 5.6] (:doubles p))) - (is (= [(float 0.01) (float 0.02) (float 0.03)] (:floats p))) - ) - (let [p (conj p {:tags ["different"]})] - (is (= ["different"] (:tags p)))) - (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] - (is (= ["little" "yellow" "different"] (:tags p))) - (is (= "very" (:label p)))))) - - (testing "append" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] - (let [p (append p {:label "bar"})] - (is (= 5 (:id p))) - (is (= "bar" (:label p))) - (is (= ["little" "yellow"] (:tags p)))) - (let [p (append p {:tags ["different"]})] - (is (= ["little" "yellow" "different"] (:tags p)))) - (let [p (append p {:tags ["different"] :label "very"})] - (is (= ["little" "yellow" "different"] (:tags p))) - (is (= "very" (:label p)))) - (let [p (append p {:doubles [3.4] :floats [0.02]})] - (is (= [1.2 3.4] (:doubles p))) - (is (= [(float 0.01) (float 0.02)] (:floats p)))))) - - (testing "assoc" - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] - (let [p (assoc p :label "baz" :tags ["nuprin"])] - (is (= ["nuprin"] (:tags p))) - (is (= "baz" (:label p)))) - (let [p (assoc p "responses" [:yes :no :maybe :no "yes"])] - (is (= [:yes :no :maybe :no :yes] (:responses p)))) - (let [p (assoc p :tags "aspirin")] - (is (= ["aspirin"] (:tags p)))) - (let [p (assoc p :foo-by-id {3 {:label "three"} 2 {:label "two"}})] - (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))))) - - (testing "dissoc" - (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] - (let [p (dissoc p :label :tags)] - (is (= [] (:tags p))) - (is (= nil (:label p)))))) - - (testing "string keys" - (let [p (protobuf Foo "id" 5 "label" "rad")] - (is (= 5 (p :id))) - (is (= 5 (p "id"))) - (is (= "rad" (p :label))) - (is (= "rad" (p "label"))) - (let [p (conj p {"tags" ["check" "it" "out"]})] - (is (= ["check" "it" "out"] (p :tags))) - (is (= ["check" "it" "out"] (p "tags")))))) - - (testing "append bytes" - (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) - q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) - r (protobuf Foo :label "bad") - s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) - t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] - (is (= 43 (s :id))) ; make sure an explicit default overwrites on append - (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append - (is (= "rad" (s :label))) - (is (= "bad" (t :label))) - (is (= ["sweet"] (t :tags))) - (is (= ["sweet" "savory"] (s :tags))) - (is (= #{"foo" "bar" "baz"} (p :tag-set))) - (is (= #{"bap" "baz"} (s :tag-set))))) - - (testing "coercing doubles and floats" - (let [p (protobuf Foo :lat 5 :long 6)] - (is (= 5.0 (p :lat))) - (is (= 6.0 (p :long)))) - (let [p (protobuf Foo :lat (float 5.0) :long (double 6.0))] - (is (= 5.0 (p :lat))) - (is (= 6.0 (p :long)))) - )) - -(deftest protobuf-extended - - (testing "create" - (let [p (protobuf Foo :id 5 :tag-set #{"little" "yellow"} :attr-map {"size" "little", "color" "yellow", "style" "different"})] - (is (= #{"little" "yellow"} (:tag-set p))) - (is (associative? (:attr-map p))) - (is (= "different" (get-in p [:attr-map "style"]))) - (is (= "little" (get-in p [:attr-map "size" ]))) - (is (= "yellow" (get-in p [:attr-map "color"])))) - (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five"}, 6 {:label "six"}})] - (let [five ((p :foo-by-id) 5) - six ((p :foo-by-id) 6)] - (is (= 5 (five :id))) - (is (= "five" (five :label))) - (is (= 6 (six :id))) - (is (= "six" (six :label)))))) - - (testing "map-by with required field" - (let [p (protobuf Foo :id 1 :item-map {"foo" {:exists true} "bar" {:exists false}})] - (is (= "foo" (get-in p [:item-map "foo" :item]))) - (is (= "bar" (get-in p [:item-map "bar" :item]))))) - - (testing "conj" - (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] - (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] - (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) - (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) - (is (= nil (get-in p [:foo-by-id 6 :label]))))))) - -(deftest protofields-and-defaults - - (testing "protofields" - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, - :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil, :lat nil, :long nil}] - (is (= fields (protofields Foo))) - (is (= fields (protofields clojure.protobuf.Test$Foo))))) - - (testing "protodefault" - (is (= 43 (protodefault Foo :id))) - (is (= 0.0 (protodefault Foo :lat))) - (is (= 0.0 (protodefault Foo :long))) - (is (= "" (protodefault Foo :label))) - (is (= [] (protodefault Foo :tags))) - (is (= nil (protodefault Foo :parent))) - (is (= [] (protodefault Foo :responses))) - (is (= #{} (protodefault Foo :tag-set))) - (is (= {} (protodefault Foo :foo-by-id))) - (is (= {} (protodefault Foo :groups))) - (is (= {} (protodefault Foo :item-map))) - (is (= {} (protodefault clojure.protobuf.Test$Foo :groups))))) +(deftest test-conj + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] + (let [p (conj p {:label "bar"})] + (is (= 5 (:id p))) + (is (= "bar" (:label p))) + (is (= ["little" "yellow"] (:tags p))) + (is (= [1.2 3.4 5.6] (:doubles p))) + (is (= [(float 0.01) (float 0.02) (float 0.03)] (:floats p))) + ) + (let [p (conj p {:tags ["different"]})] + (is (= ["different"] (:tags p)))) + (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] + (is (= ["little" "yellow" "different"] (:tags p))) + (is (= "very" (:label p)))))) + +(deftest test-append + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] + (let [p (append p {:label "bar"})] + (is (= 5 (:id p))) + (is (= "bar" (:label p))) + (is (= false (:deleted p))) + (is (= ["little" "yellow"] (:tags p)))) + (let [p (append (assoc p :deleted true) p)] + (is (= true (:deleted p)))) + (let [p (append p {:tags ["different"]})] + (is (= ["little" "yellow" "different"] (:tags p)))) + (let [p (append p {:tags ["different"] :label "very"})] + (is (= ["little" "yellow" "different"] (:tags p))) + (is (= "very" (:label p)))) + (let [p (append p {:doubles [3.4] :floats [0.02]})] + (is (= [1.2 3.4] (:doubles p))) + (is (= [(float 0.01) (float 0.02)] (:floats p)))))) + +(deftest test-adjoin + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] + (let [p (adjoin p {:label "bar"})] + (is (= 5 (:id p))) + (is (= "bar" (:label p))) + (is (= false (:deleted p))) + (is (= ["little" "yellow"] (:tags p)))) + (let [p (adjoin (assoc p :deleted true) p)] + (is (= false (:deleted p)))) + (let [p (adjoin p {:tags ["different"]})] + (is (= ["little" "yellow" "different"] (:tags p)))) + (let [p (adjoin p {:tags ["different"] :label "very"})] + (is (= ["little" "yellow" "different"] (:tags p))) + (is (= "very" (:label p)))) + (let [p (adjoin p {:doubles [3.4] :floats [0.02]})] + (is (= [1.2 3.4] (:doubles p))) + (is (= [(float 0.01) (float 0.02)] (:floats p)))))) + +(deftest test-assoc + (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] + (let [p (assoc p :label "baz" :tags ["nuprin"])] + (is (= ["nuprin"] (:tags p))) + (is (= "baz" (:label p)))) + (let [p (assoc p "responses" [:yes :no :maybe :no "yes"])] + (is (= [:yes :no :maybe :no :yes] (:responses p)))) + (let [p (assoc p :tags "aspirin")] + (is (= ["aspirin"] (:tags p)))) + (let [p (assoc p :foo-by-id {3 {:label "three"} 2 {:label "two"}})] + (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))))) + +(deftest test-dissoc + (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] + (let [p (dissoc p :label :tags)] + (is (= nil (:tags p))) + (is (= nil (:label p)))))) + +(deftest test-string-keys + (let [p (protobuf Foo "id" 5 "label" "rad")] + (is (= 5 (p :id))) + (is (= 5 (p "id"))) + (is (= "rad" (p :label))) + (is (= "rad" (p "label"))) + (let [p (conj p {"tags" ["check" "it" "out"]})] + (is (= ["check" "it" "out"] (p :tags))) + (is (= ["check" "it" "out"] (p "tags")))))) + +(deftest test-append-bytes + (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) + q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) + r (protobuf Foo :label "bad") + s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) + t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] + (is (= 43 (s :id))) ; make sure an explicit default overwrites on append + (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append + (is (= "rad" (s :label))) + (is (= "bad" (t :label))) + (is (= ["sweet"] (t :tags))) + (is (= ["sweet" "savory"] (s :tags))) + (is (= #{"foo" "bar" "baz"} (p :tag-set))) + (is (= #{"bap" "baz"} (s :tag-set))))) + +(deftest test-coercing + (let [p (protobuf Foo :lat 5 :long 6)] + (is (= 5.0 (p :lat))) + (is (= 6.0 (p :long)))) + (let [p (protobuf Foo :lat (float 5.0) :long (double 6.0))] + (is (= 5.0 (p :lat))) + (is (= 6.0 (p :long))))) + +(deftest test-create + (let [p (protobuf Foo :id 5 :tag-set #{"little" "yellow"} :attr-map {"size" "little", "color" "yellow", "style" "different"})] + (is (= #{"little" "yellow"} (:tag-set p))) + (is (associative? (:attr-map p))) + (is (= "different" (get-in p [:attr-map "style"]))) + (is (= "little" (get-in p [:attr-map "size" ]))) + (is (= "yellow" (get-in p [:attr-map "color"])))) + (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five"}, 6 {:label "six"}})] + (let [five ((p :foo-by-id) 5) + six ((p :foo-by-id) 6)] + (is (= 5 (five :id))) + (is (= "five" (five :label))) + (is (= 6 (six :id))) + (is (= "six" (six :label)))))) + +(deftest test-map-by-with-required-field + (let [p (protobuf Foo :id 1 :item-map {"foo" {:exists true} "bar" {:exists false}})] + (is (= "foo" (get-in p [:item-map "foo" :item]))) + (is (= "bar" (get-in p [:item-map "bar" :item]))))) + +(deftest test-conj + (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] + (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] + (is (= #{"prime" "odd"} (get-in p [:foo-by-id 5 :tag-set]))) + (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) + (is (= nil (get-in p [:foo-by-id 6 :label])))))) + +(deftest test-protofields + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :deleted nil, + :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil, :lat nil, :long nil}] + (is (= fields (protofields Foo))) + (is (= fields (protofields clojure.protobuf.Test$Foo))))) + +(deftest test-protodefault + (is (= 43 (protodefault Foo :id))) + (is (= 0.0 (protodefault Foo :lat))) + (is (= 0.0 (protodefault Foo :long))) + (is (= "" (protodefault Foo :label))) + (is (= [] (protodefault Foo :tags))) + (is (= nil (protodefault Foo :parent))) + (is (= [] (protodefault Foo :responses))) + (is (= #{} (protodefault Foo :tag-set))) + (is (= {} (protodefault Foo :foo-by-id))) + (is (= {} (protodefault Foo :groups))) + (is (= {} (protodefault Foo :item-map))) + (is (= false (protodefault Foo :deleted))) + (is (= {} (protodefault clojure.protobuf.Test$Foo :groups)))) (deftest use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] From a14c93c61a996e3f196a8e92b581faf8fe0e8d14 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 15 Apr 2011 18:37:56 -0700 Subject: [PATCH 115/525] clean up seq, make explicit defaults show in seq and valAt --- proto/clojure/protobuf/test.proto | 1 + src/clj/protobuf.clj | 2 +- .../protobuf/PersistentProtocolBufferMap.java | 71 +++++++++---------- test/protobuf_test.clj | 6 +- 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 6781bd9..426dd57 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -28,6 +28,7 @@ message Foo { repeated Foo foo_by_id = 12 [(map_by) = "id"]; repeated Group groups = 13 [(map) = true]; repeated Item item_map = 14 [(map_by) = "item"]; + optional bool deleted = 15 [default = false]; } message Pair { diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 78e10a8..3eb4ac2 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -50,7 +50,7 @@ (into {} (for [^Descriptors$FieldDescriptor field (.getFields type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (keyword (PersistentProtocolBufferMap/normalize (.getName field)))] + field-name (keyword (PersistentProtocolBufferMap/intern (.getName field)))] [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) (defn protobuf-load diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 0e498a4..18d59d7 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -201,16 +201,22 @@ protected DynamicMessage.Builder builder() { static boolean use_underscores = false; static public void setUseUnderscores(boolean val) { use_underscores = val; + field_name_to_keyword.clear(); enum_to_keyword.clear(); map_field_by.clear(); } - static public String normalize(String s) { - s = s.toLowerCase(); - if (!use_underscores) { - s = s.replaceAll("_","-"); + static ConcurrentHashMap field_name_to_keyword = + new ConcurrentHashMap(); + static public Keyword intern(String name) { + Keyword keyword = field_name_to_keyword.get(name); + if (keyword == null) { + name = name.toLowerCase(); + if (!use_underscores) name = name.replaceAll("_","-"); + keyword = Keyword.intern(Symbol.intern(name)); + field_name_to_keyword.putIfAbsent(name, keyword); } - return s; + return keyword; } static ConcurrentHashMap enum_to_keyword = @@ -218,8 +224,7 @@ static public String normalize(String s) { static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { Keyword keyword = enum_to_keyword.get(enum_value); if (keyword == null) { - String name = normalize(enum_value.getName()); - keyword = Keyword.intern(Symbol.intern(name)); + keyword = intern(enum_value.getName()); enum_to_keyword.putIfAbsent(enum_value, keyword); } return keyword; @@ -233,9 +238,7 @@ static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { Keyword keyword = map_field_by.get(field); if (keyword == null) { String name = field.getOptions().getExtension(Extensions.mapBy); - - name = normalize(name); - keyword = Keyword.intern(Symbol.intern(name)); + keyword = intern(name); map_field_by.putIfAbsent(field, keyword); } return keyword == k_null ? null : keyword; @@ -459,7 +462,8 @@ public IMapEntry entryAt(Object key) { public Object valAt(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return null; - if (!field.isRepeated() && !message().hasField(field)) return null; + if (field.isRepeated() && message().getRepeatedFieldCount(field) == 0) return null; + if (!field.isRepeated() && !field.hasDefaultValue() && !message().hasField(field)) return null; return fromProtoValue(field, message().getField(field)); } @@ -486,7 +490,7 @@ public IPersistentCollection cons(Object o) { if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry) o; addField(builder, e.getKey(), e.getValue()); - } else if (o instanceof IPersistentVector) { + } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector) o; if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); addField(builder, v.nth(0), v.nth(1)); @@ -528,7 +532,7 @@ public int count() { } public ISeq seq() { - return Seq.create(message()); + return Seq.create(null, this, RT.seq(def.type.getFields())); } public IPersistentCollection empty() { @@ -538,43 +542,38 @@ public IPersistentCollection empty() { } static class Seq extends ASeq { - final Map map; - final Descriptors.FieldDescriptor[] fields; - final int i; - - static public Seq create(DynamicMessage message) { - Map map = message.getAllFields(); - if (map.size() == 0) return null; - - Descriptors.FieldDescriptor[] fields = new Descriptors.FieldDescriptor[map.size()]; - fields = (Descriptors.FieldDescriptor[]) map.keySet().toArray(fields); - return new Seq(null, map, fields, 0); + final PersistentProtocolBufferMap proto; + final MapEntry first; + final ISeq fields; + + static public Seq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ + for (ISeq s = fields; s != null; s = s.next()) { + Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); + Keyword key = intern(field.getName()); + Object val = proto.valAt(key); + if (val != null) return new Seq(meta, proto, new MapEntry(key, val), s); + } + return null; } - protected Seq(IPersistentMap meta, Map map, Descriptors.FieldDescriptor[] fields, int i){ + protected Seq(IPersistentMap meta, PersistentProtocolBufferMap proto, MapEntry first, ISeq fields){ super(meta); - this.map = map; + this.proto = proto; + this.first = first; this.fields = fields; - this.i = i; } public Obj withMeta(IPersistentMap meta) { - if(meta != meta()) return new Seq(meta, map, fields, i); + if (meta != meta()) return new Seq(meta, proto, first, fields); return this; } public Object first() { - if (i == fields.length) return null; - Descriptors.FieldDescriptor field = fields[i]; - String name = normalize(field.getName()); - Keyword key = Keyword.intern(Symbol.intern(name)); - Object val = PersistentProtocolBufferMap.fromProtoValue(field, map.get(field)); - return new MapEntry(key, val); + return first; } public ISeq next() { - if (i + 1 < fields.length) return new Seq(meta(), map, fields, i + 1); - return null; + return create(meta(), proto, fields.next()); } } } \ No newline at end of file diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 3e0f4da..1492cee 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -162,14 +162,14 @@ (deftest use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] - (is (= '(:responses :tag-set) (keys p))) + (is (= '(:id :responses :tag-set :deleted) (keys p))) (is (= [:yes :not-sure :maybe :not-sure :no] (:responses p))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores true) - (is (= '(:responses :tag_set) (keys p))) + (is (= '(:id :responses :tag_set :deleted) (keys p))) (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, + (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, :deleted nil, :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil, :item_map nil, :lat nil, :long nil}] (is (= fields (protofields Foo)))) From 8e0baeaaff025a11d013529c023a7836e6dc5dbd Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 15 Apr 2011 18:38:47 -0700 Subject: [PATCH 116/525] add adjoin method (already in tests) --- project.clj | 2 +- src/clj/protobuf.clj | 7 ++++++- .../clojure/protobuf/PersistentProtocolBufferMap.java | 11 ++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index 7003d9e..a04127e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.3.4" +(defproject clojure-protobuf "0.4.0-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"]] :tasks [protobuf.tasks] diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 3eb4ac2..e90921d 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -91,4 +91,9 @@ (defn append "Merge the given map into the protobuf. Equivalent to appending the byte representations." [^PersistentProtocolBufferMap p map] - (.append p map)) \ No newline at end of file + (.append p map)) + +(defn adjoin + "Merge the given map into the protobuf. Like append, except default values in map will be used if it i." + [^PersistentProtocolBufferMap p map] + (.adjoin p map)) \ No newline at end of file diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 18d59d7..65b4259 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -143,7 +143,7 @@ static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStrea } } - static public PersistentProtocolBufferMap construct(Def def, IPersistentMap keyvals) { + static public PersistentProtocolBufferMap construct(Def def, Object keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); return (PersistentProtocolBufferMap) protobuf.cons(keyvals); } @@ -513,6 +513,15 @@ public PersistentProtocolBufferMap append(IPersistentMap map) { return new PersistentProtocolBufferMap(meta(), def, builder().mergeFrom(proto.message())); } + public PersistentProtocolBufferMap adjoin(IPersistentMap map) { + if (map instanceof PersistentProtocolBufferMap) { + PersistentProtocolBufferMap proto = (PersistentProtocolBufferMap) map; + return append(construct(def, proto.seq())); + } else { + return append(map); + } + } + public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return this; From 518e88a4919023ce0aa8cc6d62fc77db2676a0ce Mon Sep 17 00:00:00 2001 From: ninjudd Date: Fri, 15 Apr 2011 19:40:35 -0700 Subject: [PATCH 117/525] use ordered-set --- project.clj | 3 ++- src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index a04127e..c00a7d2 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,6 @@ (defproject clojure-protobuf "0.4.0-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[clojure "1.2.0"]] + :dependencies [[clojure "1.2.0"] + [ordered-set "0.2.2"]] :tasks [protobuf.tasks] :jar-files ["proto"]) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 65b4259..66881ca 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -291,7 +291,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); - ITransientSet set = (ITransientSet) PersistentHashSet.EMPTY.asTransient(); + ITransientSet set = (ITransientSet) PersistentOrderedSet.EMPTY.asTransient(); while (iterator.hasNext()) { DynamicMessage message = (DynamicMessage) iterator.next(); Object item = fromProtoValue(item_field, message.getField(item_field)); From 9733802c38902df24762be3f079975dae93bb10e Mon Sep 17 00:00:00 2001 From: ninjudd Date: Wed, 18 May 2011 18:45:16 -0700 Subject: [PATCH 118/525] version 0.4.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c00a7d2..472e519 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.0-SNAPSHOT" +(defproject clojure-protobuf "0.4.0" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 01f5e5af2b6ffbba465e902a0d22cbb06e681fcd Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 23 May 2011 18:55:41 -0700 Subject: [PATCH 119/525] fix map_by bug when value has string key which is overridden --- proto/clojure/protobuf/test.proto | 3 ++- .../protobuf/PersistentProtocolBufferMap.java | 5 +++-- test/protobuf_test.clj | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 426dd57..fa48803 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -28,7 +28,8 @@ message Foo { repeated Foo foo_by_id = 12 [(map_by) = "id"]; repeated Group groups = 13 [(map) = true]; repeated Item item_map = 14 [(map_by) = "item"]; - optional bool deleted = 15 [default = false]; + repeated Pair pair_map = 15 [(map_by) = "key"]; + optional bool deleted = 16 [default = false]; } message Pair { diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 66881ca..d53957b 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -395,8 +395,9 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) for (ISeq s = RT.seq(val); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); IPersistentMap map = (IPersistentMap) e.getValue(); - Object value = toProtoValue(field, map.assoc(map_field_by, e.getKey())); - builder.addRepeatedField(field, value); + Object k = e.getKey(); + Object v = toProtoValue(field, map.assoc(map_field_by, k).assoc(map_field_by.getName(), k)); + builder.addRepeatedField(field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(val); s != null; s = s.next()) { diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 1492cee..3fb3197 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -132,6 +132,14 @@ (is (= "foo" (get-in p [:item-map "foo" :item]))) (is (= "bar" (get-in p [:item-map "bar" :item]))))) +(deftest test-map-by-with-inconsistent-keys + (let [p (protobuf Foo :pair-map {"foo" {"key" "bar" "val" "hmm"}})] + (is (= "hmm" (get-in p [:pair-map "foo" :val]))) + (is (= nil (get-in p [:pair-map "bar" :val])))) + (let [p (protobuf Foo :pair-map {"foo" {:key "bar" :val "hmm"}})] + (is (= "hmm" (get-in p [:pair-map "foo" :val]))) + (is (= nil (get-in p [:pair-map "bar" :val]))))) + (deftest test-conj (let [p (protobuf Foo :id 1 :foo-by-id {5 {:label "five", :tag-set ["odd"]}, 6 {:label "six" :tags ["even"]}})] (let [p (conj p {:foo-by-id {5 {:tag-set ["prime" "odd"]} 6 {:tags ["odd"]}}})] @@ -141,7 +149,8 @@ (deftest test-protofields (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :deleted nil, - :attr-map nil, :foo-by-id nil, :groups nil, :doubles nil, :floats nil, :item-map nil, :lat nil, :long nil}] + :attr-map nil, :foo-by-id nil, :pair-map nil, :groups nil, :doubles nil, :floats nil, :item-map nil, + :lat nil, :long nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) @@ -170,7 +179,8 @@ (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, :deleted nil, - :attr_map nil, :foo_by_id nil, :groups nil, :doubles nil, :floats nil, :item_map nil, :lat nil, :long nil}] + :attr_map nil, :foo_by_id nil, :pair_map nil, :groups nil, :doubles nil, :floats nil, :item_map nil, + :lat nil, :long nil}] (is (= fields (protofields Foo)))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) From 6e52dae079840b01ebea9437924997f03120eb93 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 23 May 2011 18:56:43 -0700 Subject: [PATCH 120/525] version 0.4.1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 472e519..0eae3e4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.0" +(defproject clojure-protobuf "0.4.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 937fb0aa2f0dd7be69cd933f0aa74700bdd383d9 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Mon, 23 May 2011 21:50:20 -0700 Subject: [PATCH 121/525] add count extension type --- proto/clojure/protobuf/extensions.proto | 5 ++--- proto/clojure/protobuf/test.proto | 17 ++++++++++------- .../protobuf/PersistentProtocolBufferMap.java | 6 ++++++ test/protobuf_test.clj | 19 +++++++++++++++++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/proto/clojure/protobuf/extensions.proto b/proto/clojure/protobuf/extensions.proto index db4908e..2028a2f 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -4,10 +4,9 @@ option java_package = "clojure.protobuf"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { - optional bool set = 52001; + optional bool set = 52001; optional bool map = 52002; optional string map_by = 52003; - // optional bool sorted = 52005; - // optional bool multi = 52006; + optional bool count = 52004; optional string meta = 52010; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index fa48803..f23332a 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -16,6 +16,9 @@ message Foo { optional double lat = 8; optional float long = 9; + repeated uint32 count_int = 10 [(count) = true]; + repeated double count_double = 11 [(count) = true]; + enum Response { YES = 0; NO = 1; @@ -23,13 +26,13 @@ message Foo { NOT_SURE = 3; } - repeated Item tag_set = 10 [(set) = true]; - repeated Pair attr_map = 11 [(map) = true]; - repeated Foo foo_by_id = 12 [(map_by) = "id"]; - repeated Group groups = 13 [(map) = true]; - repeated Item item_map = 14 [(map_by) = "item"]; - repeated Pair pair_map = 15 [(map_by) = "key"]; - optional bool deleted = 16 [default = false]; + repeated Item tag_set = 20 [(set) = true]; + repeated Pair attr_map = 21 [(map) = true]; + repeated Foo foo_by_id = 22 [(map_by) = "id"]; + repeated Group groups = 23 [(map) = true]; + repeated Item item_map = 24 [(map_by) = "item"]; + repeated Pair pair_map = 25 [(map_by) = "key"]; + optional bool deleted = 26 [default = false]; } message Pair { diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index d53957b..a51b344 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -268,6 +268,12 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } return map.persistent(); + } else if (options.getExtension(Extensions.count)) { + Object count = iterator.next(); + while (iterator.hasNext()) { + count = Numbers.add(count, iterator.next()); + } + return count; } else if (options.getExtension(Extensions.map)) { Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 3fb3197..c9a4dee 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -147,10 +147,25 @@ (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) (is (= nil (get-in p [:foo-by-id 6 :label])))))) +(deftest test-counts + (let [p (protobuf Foo :count-int 5)] + (is (= 5 (get p :count-int))) + (let [p (append p {:count-int 2})] + (is (= 7 (get p :count-int))) + (let [p (append p {:count-int -8})] + (is (= -1 (get p :count-int)))))) + (let [p (protobuf Foo :count-double 5.0)] + (is (= 5.0 (get p :count-double))) + (let [p (append p {:count-double -2.4})] + (is (= 2.6 (get p :count-double))) + (let [p (append p {:count-double 4.06})] + (is (= 6.66 (get p :count-double))))))) + + (deftest test-protofields (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :deleted nil, :attr-map nil, :foo-by-id nil, :pair-map nil, :groups nil, :doubles nil, :floats nil, :item-map nil, - :lat nil, :long nil}] + :count-int nil, :count-double nil, :lat nil, :long nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) @@ -180,7 +195,7 @@ (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, :deleted nil, :attr_map nil, :foo_by_id nil, :pair_map nil, :groups nil, :doubles nil, :floats nil, :item_map nil, - :lat nil, :long nil}] + :count_int nil, :count_double nil, :lat nil, :long nil}] (is (= fields (protofields Foo)))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) From aefdf252c5aa6e3a67909d74ce2807f9f6d6c77e Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 24 May 2011 13:22:57 -0700 Subject: [PATCH 122/525] rename count field option to counter since field cannot have the same name as an option --- proto/clojure/protobuf/extensions.proto | 10 +++++----- proto/clojure/protobuf/test.proto | 4 ++-- .../clojure/protobuf/PersistentProtocolBufferMap.java | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/proto/clojure/protobuf/extensions.proto b/proto/clojure/protobuf/extensions.proto index 2028a2f..2e89a8f 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -4,9 +4,9 @@ option java_package = "clojure.protobuf"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { - optional bool set = 52001; - optional bool map = 52002; - optional string map_by = 52003; - optional bool count = 52004; - optional string meta = 52010; + optional bool set = 52001; + optional bool map = 52002; + optional string map_by = 52003; + optional bool counter = 52004; + optional string meta = 52010; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index f23332a..b18ec5c 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -16,8 +16,8 @@ message Foo { optional double lat = 8; optional float long = 9; - repeated uint32 count_int = 10 [(count) = true]; - repeated double count_double = 11 [(count) = true]; + repeated int32 count_int = 10 [(counter) = true]; + repeated double count_double = 11 [(counter) = true]; enum Response { YES = 0; diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index a51b344..e65be01 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -268,7 +268,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } return map.persistent(); - } else if (options.getExtension(Extensions.count)) { + } else if (options.getExtension(Extensions.counter)) { Object count = iterator.next(); while (iterator.hasNext()) { count = Numbers.add(count, iterator.next()); From c1573f90e8f472c442389dc539732386ce3fb038 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 24 May 2011 13:24:49 -0700 Subject: [PATCH 123/525] version 0.4.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0eae3e4..2fcdde8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.1" +(defproject clojure-protobuf "0.4.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 6f5499dc55f991ef4474821a939cf585553f5d65 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 24 May 2011 17:00:55 -0700 Subject: [PATCH 124/525] append messages within map_by. add test for nested counters --- proto/clojure/protobuf/test.proto | 9 ++++-- .../protobuf/PersistentProtocolBufferMap.java | 2 +- test/protobuf_test.clj | 28 +++++++++---------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index b18ec5c..a0843fa 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -16,8 +16,7 @@ message Foo { optional double lat = 8; optional float long = 9; - repeated int32 count_int = 10 [(counter) = true]; - repeated double count_double = 11 [(counter) = true]; + repeated Count counts = 10 [(map_by) = "key"]; enum Response { YES = 0; @@ -50,6 +49,12 @@ message Item { required bool exists = 2 [default = true]; } +message Count { + required string key = 1; + repeated int32 i = 2 [(counter) = true]; + repeated double d = 3 [(counter) = true]; +} + message ErrorMsg { required sint32 code = 1; optional string data = 2; diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index e65be01..472034d 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -262,7 +262,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Object key = val.valAt(map_field_by); PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(key); if (existing != null) { - map.assoc(key, existing.cons(val)); + map.assoc(key, existing.append(val)); } else { map.assoc(key, val); } diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index c9a4dee..982e32a 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -148,24 +148,22 @@ (is (= nil (get-in p [:foo-by-id 6 :label])))))) (deftest test-counts - (let [p (protobuf Foo :count-int 5)] - (is (= 5 (get p :count-int))) - (let [p (append p {:count-int 2})] - (is (= 7 (get p :count-int))) - (let [p (append p {:count-int -8})] - (is (= -1 (get p :count-int)))))) - (let [p (protobuf Foo :count-double 5.0)] - (is (= 5.0 (get p :count-double))) - (let [p (append p {:count-double -2.4})] - (is (= 2.6 (get p :count-double))) - (let [p (append p {:count-double 4.06})] - (is (= 6.66 (get p :count-double))))))) - + (let [p (protobuf Foo :counts {"foo" {:i 5 :d 5.0}})] + (is (= 5 (get-in p [:counts "foo" :i]))) + (is (= 5.0 (get-in p [:counts "foo" :d]))) + (let [p (append p {:counts {"foo" {:i 2 :d -2.4} "bar" {:i 99}}})] + (is (= 7 (get-in p [:counts "foo" :i]))) + (is (= 2.6 (get-in p [:counts "foo" :d]))) + (is (= 99 (get-in p [:counts "bar" :i]))) + (let [p (append p {:counts {"foo" {:i -8 :d 4.06} "bar" {:i -66}}})] + (is (= -1 (get-in p [:counts "foo" :i]))) + (is (= 6.66 (get-in p [:counts "foo" :d]))) + (is (= 33 (get-in p [:counts "bar" :i]))))))) (deftest test-protofields (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :deleted nil, :attr-map nil, :foo-by-id nil, :pair-map nil, :groups nil, :doubles nil, :floats nil, :item-map nil, - :count-int nil, :count-double nil, :lat nil, :long nil}] + :counts nil, :lat nil, :long nil}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) @@ -195,7 +193,7 @@ (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, :deleted nil, :attr_map nil, :foo_by_id nil, :pair_map nil, :groups nil, :doubles nil, :floats nil, :item_map nil, - :count_int nil, :count_double nil, :lat nil, :long nil}] + :counts nil, :lat nil, :long nil}] (is (= fields (protofields Foo)))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) From 108c69bca52b0d2005067c9559dbc243db4009c2 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 24 May 2011 17:01:05 -0700 Subject: [PATCH 125/525] test cleanup --- test/protobuf_test.clj | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 982e32a..789cfbd 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -182,7 +182,7 @@ (is (= false (protodefault Foo :deleted))) (is (= {} (protodefault clojure.protobuf.Test$Foo :groups)))) -(deftest use-underscores +(deftest test-use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] (is (= '(:id :responses :tag-set :deleted) (keys p))) (is (= [:yes :not-sure :maybe :not-sure :no] (:responses p))) @@ -198,13 +198,15 @@ (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) -(deftest protobuf-nested-message - (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))])) +(deftest test-protobuf-nested-message + (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))] + (is (= "abc" (get-in p [:error :data]))))) -(deftest protobuf-nil-field - (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))])) +(deftest test-protobuf-nested-null-field + (let [p (protobuf Response :ok true :error (protobuf ErrorMsg :code -10 :data nil))] + (is (:ok p)))) -(deftest protobuf-seq-and-write-protobuf +(deftest test-protobuf-seq-and-write-protobuf (let [in (PipedInputStream.) out (PipedOutputStream. in) foo (protobuf Foo :id 1 :label "foo") From 296caf3bedd28bdd17b590300bc0ce33cb896f1d Mon Sep 17 00:00:00 2001 From: ninjudd Date: Tue, 24 May 2011 17:01:42 -0700 Subject: [PATCH 126/525] version 0.4.3 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2fcdde8..af5d19d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.2" +(defproject clojure-protobuf "0.4.3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From fc5413e367e886945cbc365faefc37a6178a5bfd Mon Sep 17 00:00:00 2001 From: ninjudd Date: Thu, 9 Jun 2011 17:31:52 -0700 Subject: [PATCH 127/525] include :repeated and :type in protofields metadata --- src/clj/protobuf.clj | 11 ++++++++--- test/protobuf_test.clj | 27 ++++++++++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index e90921d..5dcba96 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -1,5 +1,6 @@ (ns protobuf - (:use [clojure.java.io :only [input-stream output-stream]]) + (:use [clojure.java.io :only [input-stream output-stream]] + [clojure.string :only [lower-case]]) (:import (clojure.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream) (java.io InputStream OutputStream))) @@ -50,8 +51,12 @@ (into {} (for [^Descriptors$FieldDescriptor field (.getFields type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (keyword (PersistentProtocolBufferMap/intern (.getName field)))] - [field-name (when-not (empty? meta-string) (read-string meta-string))]))))) + field-name (keyword (PersistentProtocolBufferMap/intern (.getName field))) + field-type (keyword (lower-case (.name (.getJavaType field))))] + [field-name (assoc (when-not (empty? meta-string) + (read-string meta-string)) + :type field-type + :repeated (.isRepeated field))]))))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 789cfbd..3ecbe3d 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -161,9 +161,23 @@ (is (= 33 (get-in p [:counts "bar" :i]))))))) (deftest test-protofields - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag-set nil, :deleted nil, - :attr-map nil, :foo-by-id nil, :pair-map nil, :groups nil, :doubles nil, :floats nil, :item-map nil, - :counts nil, :lat nil, :long nil}] + (let [fields {:parent {:repeated false, :type :message}, + :floats {:repeated true, :type :float}, + :doubles {:repeated true, :type :double}, + :counts {:repeated true, :type :message}, + :attr-map {:repeated true, :type :message}, + :tag-set {:repeated true, :type :message}, + :item-map {:repeated true, :type :message}, + :groups {:repeated true, :type :message}, + :responses {:repeated true, :type :enum}, + :lat {:repeated false, :type :double}, + :pair-map {:repeated true, :type :message}, + :foo-by-id {:repeated true, :type :message}, + :label {:repeated false, :type :string, :a 1, :b 2, :c 3}, + :id {:repeated false, :type :int}, + :long {:repeated false, :type :float}, + :deleted {:repeated false, :type :boolean}, + :tags {:repeated true, :type :string}}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) @@ -191,10 +205,9 @@ (is (= '(:id :responses :tag_set :deleted) (keys p))) (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) - (let [fields {:id nil, :label {:a 1, :b 2, :c 3}, :tags nil, :parent nil, :responses nil, :tag_set nil, :deleted nil, - :attr_map nil, :foo_by_id nil, :pair_map nil, :groups nil, :doubles nil, :floats nil, :item_map nil, - :counts nil, :lat nil, :long nil}] - (is (= fields (protofields Foo)))) + (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id + :pair_map :groups :doubles :floats :item_map :counts :lat :long} + (set (keys (protofields Foo))))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) From ac5fb9af03f787081d05a1a8dd6d1371a256bb66 Mon Sep 17 00:00:00 2001 From: ninjudd Date: Thu, 9 Jun 2011 17:33:58 -0700 Subject: [PATCH 128/525] version 0.4.4 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index af5d19d..fea5ceb 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.3" +(defproject clojure-protobuf "0.4.4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 8d4a73a5f94e40ea0bcc4142653dac8509d6d4ac Mon Sep 17 00:00:00 2001 From: ninjudd Date: Sun, 12 Jun 2011 10:23:20 -0700 Subject: [PATCH 129/525] add succession extension type and (get-raw ...) to get value bypassing extensions --- project.clj | 2 +- proto/clojure/protobuf/extensions.proto | 11 +- proto/clojure/protobuf/test.proto | 10 ++ src/clj/protobuf.clj | 7 +- .../protobuf/PersistentProtocolBufferMap.java | 147 ++++++++++-------- test/protobuf_test.clj | 25 ++- 6 files changed, 124 insertions(+), 78 deletions(-) diff --git a/project.clj b/project.clj index fea5ceb..cce2979 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.4" +(defproject clojure-protobuf "0.4.5" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] diff --git a/proto/clojure/protobuf/extensions.proto b/proto/clojure/protobuf/extensions.proto index 2e89a8f..a56dd34 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -4,9 +4,10 @@ option java_package = "clojure.protobuf"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { - optional bool set = 52001; - optional bool map = 52002; - optional string map_by = 52003; - optional bool counter = 52004; - optional string meta = 52010; + optional bool set = 52001; + optional bool map = 52002; + optional string map_by = 52003; + optional bool counter = 52004; + optional bool succession = 52005; + optional string meta = 52010; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index a0843fa..08aee2b 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -18,6 +18,8 @@ message Foo { repeated Count counts = 10 [(map_by) = "key"]; + repeated Time time = 11 [(succession) = true]; + enum Response { YES = 0; NO = 1; @@ -34,6 +36,14 @@ message Foo { optional bool deleted = 26 [default = false]; } +message Time { + optional sint32 year = 1; + optional int32 month = 2; + optional int32 day = 3; + optional int32 hour = 4; + optional int32 minute = 5; +} + message Pair { required string key = 1; required string val = 2; diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 5dcba96..d69c764 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -101,4 +101,9 @@ (defn adjoin "Merge the given map into the protobuf. Like append, except default values in map will be used if it i." [^PersistentProtocolBufferMap p map] - (.adjoin p map)) \ No newline at end of file + (.adjoin p map)) + +(defn get-raw + "Get value at key ignoring extension fields." + [^PersistentProtocolBufferMap p key] + (.getValAt p key false)) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 472034d..79b7ea4 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -249,78 +249,85 @@ static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { static Keyword k_item = Keyword.intern(Symbol.intern("item")); static Keyword k_exists = Keyword.intern(Symbol.intern("exists")); static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { + return fromProtoValue(field, value, true); + } + + static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, boolean use_extensions) { if (value instanceof List) { List values = (List) value; Iterator iterator = values.iterator(); - Keyword map_field_by = mapFieldBy(field); - DescriptorProtos.FieldOptions options = field.getOptions(); - if (map_field_by != null) { - ITransientMap map = PersistentHashMap.EMPTY.asTransient(); - while (iterator.hasNext()) { - PersistentProtocolBufferMap val = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); - Object key = val.valAt(map_field_by); - PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(key); - if (existing != null) { - map.assoc(key, existing.append(val)); - } else { - map.assoc(key, val); + if (use_extensions) { + Keyword map_field_by = mapFieldBy(field); + DescriptorProtos.FieldOptions options = field.getOptions(); + if (map_field_by != null) { + ITransientMap map = PersistentHashMap.EMPTY.asTransient(); + while (iterator.hasNext()) { + PersistentProtocolBufferMap val = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); + Object key = val.valAt(map_field_by); + PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(key); + if (existing != null) { + map.assoc(key, existing.append(val)); + } else { + map.assoc(key, val); + } } - } - return map.persistent(); - } else if (options.getExtension(Extensions.counter)) { - Object count = iterator.next(); - while (iterator.hasNext()) { - count = Numbers.add(count, iterator.next()); - } - return count; - } else if (options.getExtension(Extensions.map)) { - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); - Descriptors.FieldDescriptor val_field = def.fieldDescriptor(k_val); - - ITransientMap map = PersistentHashMap.EMPTY.asTransient(); - while (iterator.hasNext()) { - DynamicMessage message = (DynamicMessage) iterator.next(); - Object key = fromProtoValue(key_field, message.getField(key_field)); - Object val = fromProtoValue(val_field, message.getField(val_field)); - Object existing = map.valAt(key); - if (existing != null && existing instanceof IPersistentCollection) { - map.assoc(key, ((IPersistentCollection) existing).cons(val)); - } else { - map.assoc(key, val); + return map.persistent(); + } else if (options.getExtension(Extensions.counter)) { + Object count = iterator.next(); + while (iterator.hasNext()) { + count = Numbers.add(count, iterator.next()); } - } - return map.persistent(); - } else if (options.getExtension(Extensions.set)) { - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); - Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); - - ITransientSet set = (ITransientSet) PersistentOrderedSet.EMPTY.asTransient(); - while (iterator.hasNext()) { - DynamicMessage message = (DynamicMessage) iterator.next(); - Object item = fromProtoValue(item_field, message.getField(item_field)); - Boolean exists = (Boolean) message.getField(exists_field); - - if (exists) { - set.conj(item); - } else { - try { - set.disjoin(item); - } catch (Exception e) { - e.printStackTrace(); + return count; + } else if (options.getExtension(Extensions.succession)) { + return fromProtoValue(field, values.get(values.size() - 1)); + } else if (options.getExtension(Extensions.map)) { + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); + Descriptors.FieldDescriptor val_field = def.fieldDescriptor(k_val); + + ITransientMap map = PersistentHashMap.EMPTY.asTransient(); + while (iterator.hasNext()) { + DynamicMessage message = (DynamicMessage) iterator.next(); + Object key = fromProtoValue(key_field, message.getField(key_field)); + Object val = fromProtoValue(val_field, message.getField(val_field)); + Object existing = map.valAt(key); + if (existing != null && existing instanceof IPersistentCollection) { + map.assoc(key, ((IPersistentCollection) existing).cons(val)); + } else { + map.assoc(key, val); } } + return map.persistent(); + } else if (options.getExtension(Extensions.set)) { + Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); + Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); + + ITransientSet set = (ITransientSet) PersistentOrderedSet.EMPTY.asTransient(); + while (iterator.hasNext()) { + DynamicMessage message = (DynamicMessage) iterator.next(); + Object item = fromProtoValue(item_field, message.getField(item_field)); + Boolean exists = (Boolean) message.getField(exists_field); + + if (exists) { + set.conj(item); + } else { + try { + set.disjoin(item); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + return set.persistent(); } - return set.persistent(); - } else { - List list = new ArrayList(values.size()); - while (iterator.hasNext()) { - list.add(fromProtoValue(field, iterator.next())); - } - return PersistentVector.create(list); } + List list = new ArrayList(values.size()); + while (iterator.hasNext()) { + list.add(fromProtoValue(field, iterator.next(), use_extensions)); + } + return PersistentVector.create(list); } else { switch (field.getJavaType()) { case ENUM: @@ -331,7 +338,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object DynamicMessage message = (DynamicMessage) value; // Total hack because getField() doesn't return an empty array for repeated messages. - if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList()); + if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList(), use_extensions); return new PersistentProtocolBufferMap(null, def, message); default: @@ -467,11 +474,7 @@ public IMapEntry entryAt(Object key) { } public Object valAt(Object key) { - Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return null; - if (field.isRepeated() && message().getRepeatedFieldCount(field) == 0) return null; - if (!field.isRepeated() && !field.hasDefaultValue() && !message().hasField(field)) return null; - return fromProtoValue(field, message().getField(field)); + return getValAt(key, true); } public Object valAt(Object key, Object notFound) { @@ -479,6 +482,14 @@ public Object valAt(Object key, Object notFound) { return (val == null) ? notFound : val; } + public Object getValAt(Object key, boolean use_extensions) { + Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + if (field == null) return null; + if (field.isRepeated() && message().getRepeatedFieldCount(field) == 0) return null; + if (!field.isRepeated() && !field.hasDefaultValue() && !message().hasField(field)) return null; + return fromProtoValue(field, message().getField(field), use_extensions); + } + public IPersistentMap assoc(Object key, Object val) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor(key); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 3ecbe3d..46a4f82 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -147,7 +147,7 @@ (is (= ["odd"] (get-in p [:foo-by-id 6 :tags]))) (is (= nil (get-in p [:foo-by-id 6 :label])))))) -(deftest test-counts +(deftest test-nested-counters (let [p (protobuf Foo :counts {"foo" {:i 5 :d 5.0}})] (is (= 5 (get-in p [:counts "foo" :i]))) (is (= 5.0 (get-in p [:counts "foo" :d]))) @@ -158,13 +158,32 @@ (let [p (append p {:counts {"foo" {:i -8 :d 4.06} "bar" {:i -66}}})] (is (= -1 (get-in p [:counts "foo" :i]))) (is (= 6.66 (get-in p [:counts "foo" :d]))) - (is (= 33 (get-in p [:counts "bar" :i]))))))) + (is (= 33 (get-in p [:counts "bar" :i]))) + (is (= [{:key "foo", :i 5, :d 5.0} + {:key "foo", :i 2, :d -2.4} + {:key "bar", :i 99} + {:key "foo", :i -8, :d 4.06} + {:key "bar", :i -66}] + (get-raw p :counts))))))) + +(deftest test-succession + (let [p (protobuf Foo :time {:year 1978 :month 11 :day 24})] + (is (= 1978 (get-in p [:time :year]))) + (is (= 11 (get-in p [:time :month]))) + (is (= 24 (get-in p [:time :day]))) + (let [p (append p {:time {:year 1974 :month 1}})] + (is (= 1974 (get-in p [:time :year]))) + (is (= 1 (get-in p [:time :month]))) + (is (= nil (get-in p [:time :day]))) + (is (= [{:year 1978, :month 11, :day 24} {:year 1974, :month 1}] + (get-raw p :time)))))) (deftest test-protofields (let [fields {:parent {:repeated false, :type :message}, :floats {:repeated true, :type :float}, :doubles {:repeated true, :type :double}, :counts {:repeated true, :type :message}, + :time {:repeated true, :type :message}, :attr-map {:repeated true, :type :message}, :tag-set {:repeated true, :type :message}, :item-map {:repeated true, :type :message}, @@ -206,7 +225,7 @@ (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id - :pair_map :groups :doubles :floats :item_map :counts :lat :long} + :pair_map :groups :doubles :floats :item_map :counts :time :lat :long} (set (keys (protofields Foo))))) (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) From e636474c464251d3df04b2b9dc01fc058763ad95 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 27 Jun 2011 15:08:50 -0700 Subject: [PATCH 130/525] depot updates --- project.clj | 2 +- src/clj/protobuf/tasks.clj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index cce2979..dc163d7 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.5" +(defproject clojure-protobuf "0.4.6" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 08527f7..0bec2dc 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -45,7 +45,7 @@ (when (installed?) (let [password (prompt-read "Password" :echo false)] (ant ExecTask {:dir srcdir :executable "sudo" :input-string (str password "\n")} - (args ["-S" "make" "uninstall"]))))) + (args ["-S" "make" "uninstall"]))))) (defn- proto-dependencies "look for lines starting with import in proto-file" [proto-file] @@ -81,7 +81,7 @@ (log "Compiling" proto "to" dest) (extract-dependencies (file "proto" proto)) (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} - (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) + (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) (catch org.apache.tools.ant.BuildException e (throw (Exception. (str "error compiling " proto)))))) (compile-java (file dest))))) @@ -102,7 +102,7 @@ (.substring (.getPath file) (inc (count (.getPath dir)))))) (deftask compile-native #{proto}) -(deftask proto #{deps install-protoc} +(deftask proto #{install-protoc} "Compile protocol buffer files located in proto dir." (if (= "clojure-protobuf" (:artifact-id *project*)) (do (invoke fetch-protoc) From 1028efbebc54f17a2422f6cec611cab7f11a2f44 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 29 Jun 2011 18:21:23 -0700 Subject: [PATCH 131/525] updates for cake depot --- project.clj | 3 ++- src/clj/protobuf/tasks.clj | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index dc163d7..5794673 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,7 @@ -(defproject clojure-protobuf "0.4.6" +(defproject clojure-protobuf "0.4.7-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] :tasks [protobuf.tasks] + :source-path ["src/clj" "src/jvm"] :jar-files ["proto"]) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 0bec2dc..9d69f23 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -84,7 +84,7 @@ (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) (catch org.apache.tools.ant.BuildException e (throw (Exception. (str "error compiling " proto)))))) - (compile-java (file dest))))) + (compile-java dest)))) (defn build-protobuf [] (ant Mkdir {:dir "proto/google/protobuf"}) From 847c3930013ac3967b98f76bc2818fafc29bbd90 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 30 Jun 2011 13:09:37 -0700 Subject: [PATCH 132/525] add support for nested subfields with protofields --- src/clj/protobuf.clj | 32 +++++++++++++++++--------------- test/protobuf_test.clj | 9 +++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index d69c764..12cd482 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -17,10 +17,15 @@ (defn protodef "Create a protodef from a string or protobuf class." - [class] - (if (or (protodef? class) (nil? class)) - class - (PersistentProtocolBufferMap$Def/create class))) + ([class] + (if (or (protodef? class) (nil? class)) + class + (PersistentProtocolBufferMap$Def/create class))) + ([class field] + (-> (protodef class) + (.fieldDescriptor field) + .getMessageType + protodef))) (defmacro defprotobuf "Helper macro for defining a protodef object." @@ -45,18 +50,15 @@ (defn protofields "Return a map of the protobuf fields to the clojure.protobuf.Extensions metadata for each field." - [type] - (let [type ^PersistentProtocolBufferMap$Def (protodef type) - type ^Descriptors$Descriptor (.getMessageType type)] + [& args] + (let [type (.getMessageType ^PersistentProtocolBufferMap$Def (apply protodef args))] (into {} - (for [^Descriptors$FieldDescriptor field (.getFields type)] - (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (keyword (PersistentProtocolBufferMap/intern (.getName field))) - field-type (keyword (lower-case (.name (.getJavaType field))))] - [field-name (assoc (when-not (empty? meta-string) - (read-string meta-string)) - :type field-type - :repeated (.isRepeated field))]))))) + (for [^Descriptors$FieldDescriptor field (.getFields ^Descriptors$Descriptor type)] + (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) + field-name (PersistentProtocolBufferMap/intern (.getName field))] + [field-name (assoc (when (seq meta-string) (read-string meta-string)) + :type (keyword (lower-case (.name (.getJavaType field)))) + :repeated (.isRepeated field))]))))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 46a4f82..889cdda 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -200,6 +200,15 @@ (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) +(deftest test-nested-protofields + (is (= {:year {:repeated false, :type :int}, + :month {:repeated false, :type :int}, + :day {:repeated false, :type :int}, + :hour {:repeated false, :type :int}, + :minute {:repeated false, :type :int}} + (protofields Foo :time)))) + + (deftest test-protodefault (is (= 43 (protodefault Foo :id))) (is (= 0.0 (protodefault Foo :lat))) From 4148afae36a70fac927fe5d842215f89111f10fa Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 30 Jun 2011 15:54:44 -0700 Subject: [PATCH 133/525] add enum :values and arbitrarily deep nested field support to protofields --- src/clj/protobuf.clj | 28 +++++++--- .../protobuf/PersistentProtocolBufferMap.java | 3 +- test/protobuf_test.clj | 55 ++++++++++--------- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 12cd482..3f2d33f 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -21,11 +21,16 @@ (if (or (protodef? class) (nil? class)) class (PersistentProtocolBufferMap$Def/create class))) - ([class field] - (-> (protodef class) - (.fieldDescriptor field) - .getMessageType - protodef))) + ([class & fields] + (loop [type (protodef class) + fields fields] + (if (empty? fields) + type + (recur (-> type + (.fieldDescriptor (first fields)) + .getMessageType + protodef) + (rest fields)))))) (defmacro defprotobuf "Helper macro for defining a protodef object." @@ -55,10 +60,15 @@ (into {} (for [^Descriptors$FieldDescriptor field (.getFields ^Descriptors$Descriptor type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (PersistentProtocolBufferMap/intern (.getName field))] - [field-name (assoc (when (seq meta-string) (read-string meta-string)) - :type (keyword (lower-case (.name (.getJavaType field)))) - :repeated (.isRepeated field))]))))) + field-name (PersistentProtocolBufferMap/intern (.getName field)) + field-type (keyword (lower-case (.name (.getJavaType field))))] + [field-name (merge (when (seq meta-string) (read-string meta-string)) + {:type field-type} + (when (.isRepeated field) + {:repeated true}) + (when (= :enum field-type) + {:values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) + (.. field getEnumType getValues)))}))]))))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index 79b7ea4..f9f29fc 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -221,7 +221,7 @@ static public Keyword intern(String name) { static ConcurrentHashMap enum_to_keyword = new ConcurrentHashMap(); - static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { + static public Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { Keyword keyword = enum_to_keyword.get(enum_value); if (keyword == null) { keyword = intern(enum_value.getName()); @@ -230,7 +230,6 @@ static protected Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_valu return keyword; } - static Keyword k_null = Keyword.intern(Symbol.intern("")); static ConcurrentHashMap map_field_by = new ConcurrentHashMap(); diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 889cdda..dd713e3 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -179,35 +179,40 @@ (get-raw p :time)))))) (deftest test-protofields - (let [fields {:parent {:repeated false, :type :message}, - :floats {:repeated true, :type :float}, - :doubles {:repeated true, :type :double}, - :counts {:repeated true, :type :message}, - :time {:repeated true, :type :message}, - :attr-map {:repeated true, :type :message}, - :tag-set {:repeated true, :type :message}, - :item-map {:repeated true, :type :message}, - :groups {:repeated true, :type :message}, - :responses {:repeated true, :type :enum}, - :lat {:repeated false, :type :double}, - :pair-map {:repeated true, :type :message}, - :foo-by-id {:repeated true, :type :message}, - :label {:repeated false, :type :string, :a 1, :b 2, :c 3}, - :id {:repeated false, :type :int}, - :long {:repeated false, :type :float}, - :deleted {:repeated false, :type :boolean}, - :tags {:repeated true, :type :string}}] + (let [fields {:floats {:type :float, :repeated true}, + :doubles {:type :double, :repeated true}, + :counts {:type :message, :repeated true}, + :time {:type :message, :repeated true}, + :attr-map {:type :message, :repeated true}, + :tag-set {:type :message, :repeated true}, + :item-map {:type :message, :repeated true}, + :groups {:type :message, :repeated true}, + :responses {:type :enum, :repeated true, :values #{:yes :no :maybe :not-sure}}, + :pair-map {:type :message, :repeated true}, + :foo-by-id {:type :message, :repeated true}, + :tags {:type :string, :repeated true}, + :label {:type :string, :a 1, :b 2, :c 3}, + :id {:type :int}, + :parent {:type :message}, + :lat {:type :double}, + :long {:type :float}, + :deleted {:type :boolean}}] (is (= fields (protofields Foo))) (is (= fields (protofields clojure.protobuf.Test$Foo))))) (deftest test-nested-protofields - (is (= {:year {:repeated false, :type :int}, - :month {:repeated false, :type :int}, - :day {:repeated false, :type :int}, - :hour {:repeated false, :type :int}, - :minute {:repeated false, :type :int}} - (protofields Foo :time)))) - + (is (= {:year {:type :int}, + :month {:type :int}, + :day {:type :int}, + :hour {:type :int}, + :minute {:type :int}} + (protofields Foo :time))) + (is (= {:year {:type :int}, + :month {:type :int}, + :day {:type :int}, + :hour {:type :int}, + :minute {:type :int}} + (protofields Foo :parent :foo-by-id :time)))) (deftest test-protodefault (is (= 43 (protodefault Foo :id))) From a3960121c0a6a1f0e7f3796792de17c2b72ec647 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 1 Jul 2011 13:16:42 -0700 Subject: [PATCH 134/525] rename project to protobuf --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5794673..97d0393 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojure-protobuf "0.4.7-SNAPSHOT" +(defproject protobuf "0.4.7-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 95d79ed58b8908253858cccb4ae39b8415e195e6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 12 Jul 2011 14:25:19 -0700 Subject: [PATCH 135/525] 0.5.0-alpha1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 97d0393..8087cf5 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.4.7-SNAPSHOT" +(defproject protobuf "0.5.0-alpha1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 76894ce08bb4c537fa2f71122ee0607db197b6c8 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 19 Jul 2011 17:12:08 -0500 Subject: [PATCH 136/525] Add cake-marginalia. --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8087cf5..0283c26 100644 --- a/project.clj +++ b/project.clj @@ -2,6 +2,7 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] - :tasks [protobuf.tasks] + :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.1"]] + :tasks [protobuf.tasks cake-marginalia.tasks] :source-path ["src/clj" "src/jvm"] :jar-files ["proto"]) From 4b37c78cd63007de208a999eebcf9c1b38c20604 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 19 Jul 2011 17:13:21 -0500 Subject: [PATCH 137/525] Add docs to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 79b5bdd..a2e3401 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ pom.xml *.jar *.class proto/google/protobuf/descriptor.proto +docs From 735a4be6416b35d2d3b692ed213be8a67fca5f7b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 18 Aug 2011 17:40:33 -0700 Subject: [PATCH 138/525] bump cake-marginalia --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 0283c26..a5579de 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] - :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.1"]] + :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.3"]] :tasks [protobuf.tasks cake-marginalia.tasks] :source-path ["src/clj" "src/jvm"] :jar-files ["proto"]) From 171dadc50280ad0a890641892b82f2a528a849aa Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 19 Aug 2011 12:04:12 -0700 Subject: [PATCH 139/525] fix proto task within protobuf project --- src/clj/protobuf/tasks.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj index 9d69f23..132b36c 100644 --- a/src/clj/protobuf/tasks.clj +++ b/src/clj/protobuf/tasks.clj @@ -104,7 +104,7 @@ (deftask compile-native #{proto}) (deftask proto #{install-protoc} "Compile protocol buffer files located in proto dir." - (if (= "clojure-protobuf" (:artifact-id *project*)) + (if (= "protobuf" (:artifact-id *project*)) (do (invoke fetch-protoc) (build-protobuf)) (protoc (or (:proto *opts*) (proto-files (file "proto")))))) From a3770a98931b77cf9d1549dc1d842b00f2df69d9 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 19 Aug 2011 13:43:17 -0700 Subject: [PATCH 140/525] add support for nullable fields --- project.clj | 2 +- proto/clojure/protobuf/extensions.proto | 18 ++- proto/clojure/protobuf/test.proto | 11 ++ .../protobuf/PersistentProtocolBufferMap.java | 110 ++++++++++-------- test/protobuf_test.clj | 32 ++++- 5 files changed, 118 insertions(+), 55 deletions(-) diff --git a/project.clj b/project.clj index a5579de..74dc6f8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-alpha1" +(defproject protobuf "0.5.0-alpha2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] diff --git a/proto/clojure/protobuf/extensions.proto b/proto/clojure/protobuf/extensions.proto index a56dd34..f72066e 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/clojure/protobuf/extensions.proto @@ -4,10 +4,16 @@ option java_package = "clojure.protobuf"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { - optional bool set = 52001; - optional bool map = 52002; - optional string map_by = 52003; - optional bool counter = 52004; - optional bool succession = 52005; - optional string meta = 52010; + optional bool set = 52001; + optional bool map = 52002; + optional string map_by = 52003; + optional bool counter = 52004; + optional bool succession = 52005; + optional string meta = 52010; + optional bool nullable = 52020; + optional string null_string = 52021; + optional sint32 null_int = 52022; + optional sint64 null_long = 52023; + optional float null_float = 52024; + optional double null_double = 52025; } diff --git a/proto/clojure/protobuf/test.proto b/proto/clojure/protobuf/test.proto index 08aee2b..7149e8d 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/clojure/protobuf/test.proto @@ -36,6 +36,17 @@ message Foo { optional bool deleted = 26 [default = false]; } +message Bar { + optional int32 int = 1 [(nullable) = true, (null_int) = -1]; + optional int64 long = 2 [(nullable) = true, (null_long) = -999999999999]; + optional float flt = 3 [(nullable) = true, (null_float) = -0.0001]; + optional double dbl = 4 [(nullable) = true, (null_double) = -0.00000001]; + optional string str = 5 [(nullable) = true, (null_string) = "NULL"]; + + repeated string label = 6 [(nullable) = true, (null_string) = "", (succession) = true]; + repeated string labels = 7 [(nullable) = true, (null_string) = ""]; +} + message Time { optional sint32 year = 1; optional int32 month = 2; diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index f9f29fc..a724aad 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -24,6 +24,7 @@ import com.google.protobuf.DescriptorProtos; import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.GeneratedMessage; public class PersistentProtocolBufferMap extends APersistentMap { public static class Def { @@ -262,13 +263,13 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object if (map_field_by != null) { ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { - PersistentProtocolBufferMap val = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); - Object key = val.valAt(map_field_by); - PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(key); + PersistentProtocolBufferMap v = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); + Object k = v.valAt(map_field_by); + PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(k); if (existing != null) { - map.assoc(key, existing.append(val)); + map.assoc(k, existing.append(v)); } else { - map.assoc(key, val); + map.assoc(k, v); } } return map.persistent(); @@ -288,13 +289,13 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { DynamicMessage message = (DynamicMessage) iterator.next(); - Object key = fromProtoValue(key_field, message.getField(key_field)); - Object val = fromProtoValue(val_field, message.getField(val_field)); - Object existing = map.valAt(key); + Object k = fromProtoValue(key_field, message.getField(key_field)); + Object v = fromProtoValue(val_field, message.getField(val_field)); + Object existing = map.valAt(k); if (existing != null && existing instanceof IPersistentCollection) { - map.assoc(key, ((IPersistentCollection) existing).cons(val)); + map.assoc(k, ((IPersistentCollection) existing).cons(v)); } else { - map.assoc(key, val); + map.assoc(k, v); } } return map.persistent(); @@ -341,21 +342,26 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object return new PersistentProtocolBufferMap(null, def, message); default: + if (use_extensions && + field.getOptions().getExtension(Extensions.nullable) && + field.getOptions().getExtension(nullExtension(field)).equals(value)) + return null; return value; } } } static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { + if (value == null && field.getOptions().getExtension(Extensions.nullable)) + value = field.getOptions().getExtension(nullExtension(field)); + switch (field.getJavaType()) { case LONG: if (value instanceof Long) return value; - Integer i = (Integer) value; - return new Long(i.longValue()); + return new Long(((Integer) value).longValue()); case INT: if (value instanceof Integer) return value; - Long l = (Long) value; - return new Integer(l.intValue()); + return new Integer(((Long) value).intValue()); case FLOAT: if (value instanceof Integer) return new Float((Integer) value * 1.0); if (value instanceof Double) return new Float((Double) value); @@ -388,23 +394,35 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v } } - protected void addField(DynamicMessage.Builder builder, Object key, Object val) { - if (key == null || val == null) return; + static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.FieldDescriptor field) { + switch (field.getJavaType()) { + case LONG: return Extensions.nullLong; + case INT: return Extensions.nullInt; + case FLOAT: return Extensions.nullFloat; + case DOUBLE: return Extensions.nullDouble; + case STRING: return Extensions.nullString; + } + return null; + } + + protected void addField(DynamicMessage.Builder builder, Object key, Object value) { + if (key == null) return; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return; + if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) return; boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { builder.clearField(field); - if (val instanceof Sequential && !set) { - for (ISeq s = RT.seq(val); s != null; s = s.next()) { - Object value = toProtoValue(field, s.first()); - builder.addRepeatedField(field, value); + if (value instanceof Sequential && !set) { + for (ISeq s = RT.seq(value); s != null; s = s.next()) { + Object v = toProtoValue(field, s.first()); + builder.addRepeatedField(field, v); } } else { Keyword map_field_by = mapFieldBy(field); if (map_field_by != null) { - for (ISeq s = RT.seq(val); s != null; s = s.next()) { + for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); IPersistentMap map = (IPersistentMap) e.getValue(); Object k = e.getKey(); @@ -412,38 +430,38 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object val) builder.addRepeatedField(field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { - for (ISeq s = RT.seq(val); s != null; s = s.next()) { + for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; - Object value = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, value); + Object v = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, v); } } else if (set) { - if (val instanceof IPersistentMap) { - for (ISeq s = RT.seq(val); s != null; s = s.next()) { + if (value instanceof IPersistentMap) { + for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_item, e.getKey(), k_exists, e.getValue()}; - Object value = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, value); + Object v = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, v); } } else { - for (ISeq s = RT.seq(val); s != null; s = s.next()) { + for (ISeq s = RT.seq(value); s != null; s = s.next()) { Object[] map = {k_item, s.first(), k_exists, true}; - Object value = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, value); + Object v = toProtoValue(field, new PersistentArrayMap(map)); + builder.addRepeatedField(field, v); } } } else { - Object value = toProtoValue(field, val); - builder.addRepeatedField(field, value); + Object v = toProtoValue(field, value); + builder.addRepeatedField(field, v); } } } else { - Object value = toProtoValue(field, val); - if (value instanceof DynamicMessage) { - value = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) value).build(); + Object v = toProtoValue(field, value); + if (v instanceof DynamicMessage) { + v = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) v).build(); } - builder.setField(field, value); + builder.setField(field, v); } } @@ -477,8 +495,8 @@ public Object valAt(Object key) { } public Object valAt(Object key, Object notFound) { - Object val = valAt(key); - return (val == null) ? notFound : val; + Object value = valAt(key); + return (value == null) ? notFound : value; } public Object getValAt(Object key, boolean use_extensions) { @@ -489,17 +507,17 @@ public Object getValAt(Object key, boolean use_extensions) { return fromProtoValue(field, message().getField(field), use_extensions); } - public IPersistentMap assoc(Object key, Object val) { + public IPersistentMap assoc(Object key, Object value) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - addField(builder, field, val); + addField(builder, field, value); return new PersistentProtocolBufferMap(meta(), def, builder); } - public IPersistentMap assocEx(Object key, Object val) throws Exception { + public IPersistentMap assocEx(Object key, Object value) throws Exception { if(containsKey(key)) throw new Exception("Key already present"); - return assoc(key, val); + return assoc(key, value); } public IPersistentCollection cons(Object o) { @@ -575,9 +593,9 @@ static class Seq extends ASeq { static public Seq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ for (ISeq s = fields; s != null; s = s.next()) { Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); - Keyword key = intern(field.getName()); - Object val = proto.valAt(key); - if (val != null) return new Seq(meta, proto, new MapEntry(key, val), s); + Keyword k = intern(field.getName()); + Object v = proto.valAt(k); + if (v != null) return new Seq(meta, proto, new MapEntry(k, v), s); } return null; } diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index dd713e3..8b4dea4 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -4,6 +4,7 @@ (:import (java.io PipedInputStream PipedOutputStream))) (defprotobuf Foo clojure.protobuf.Test Foo) +(defprotobuf Bar clojure.protobuf.Test Bar) (defprotobuf Response clojure.protobuf.Test Response) (defprotobuf ErrorMsg clojure.protobuf.Test ErrorMsg) @@ -17,8 +18,7 @@ (is (= "bar" (:label p))) (is (= ["little" "yellow"] (:tags p))) (is (= [1.2 3.4 5.6] (:doubles p))) - (is (= [(float 0.01) (float 0.02) (float 0.03)] (:floats p))) - ) + (is (= [(float 0.01) (float 0.02) (float 0.03)] (:floats p)))) (let [p (conj p {:tags ["different"]})] (is (= ["different"] (:tags p)))) (let [p (conj p {:tags ["little" "yellow" "different"] :label "very"})] @@ -178,6 +178,34 @@ (is (= [{:year 1978, :month 11, :day 24} {:year 1974, :month 1}] (get-raw p :time)))))) +(deftest test-nullable + (let [p (protobuf Bar :int 1 :long 330000000000 :flt 1.23 :dbl 9.87654321 :str "foo")] + (is (= 1 (get p :int))) + (is (= 330000000000 (get p :long))) + (is (= (float 1.23) (get p :flt))) + (is (= 9.87654321 (get p :dbl))) + (is (= "foo" (get p :str))) + (is (= [:int :long :flt :dbl :str] (keys p))) + (let [p (append p {:int nil :long nil :flt nil :dbl nil :str nil})] + (is (= nil (get p :int))) + (is (= nil (get p :long))) + (is (= nil (get p :flt))) + (is (= nil (get p :dbl))) + (is (= nil (get p :str))) + (is (= nil (keys p)))) + (testing "nullable successions" + (let [p (protobuf Bar :label "foo")] + (is (= "foo" (get p :label))) + (let [p (append p {:label nil})] + (is (= nil (get p :label))) + (is (= ["foo" ""] (get-raw p :label)))))) + (testing "repeated nullable" + (let [p (protobuf Bar :labels ["foo" "bar"])] + (is (= ["foo" "bar"] (get p :labels))) + (let [p (append p {:labels [nil]})] + (is (= ["foo" "bar" nil] (get p :labels))) + (is (= ["foo" "bar" ""] (get-raw p :labels)))))))) + (deftest test-protofields (let [fields {:floats {:type :float, :repeated true}, :doubles {:type :double, :repeated true}, From 554edcd230a573c87663b3298788ff3c7c2368fd Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 26 Aug 2011 19:00:53 -0700 Subject: [PATCH 141/525] fix pom --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 74dc6f8..0575280 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-alpha2" +(defproject protobuf "0.5.0-alpha3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From c86857bd5a5f6f7b084b3b2887e732cab7ddb889 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 20 Sep 2011 19:40:30 -0700 Subject: [PATCH 142/525] alpha4 --- project.clj | 7 ++- src/clj/protobuf/tasks.clj | 110 ------------------------------------- 2 files changed, 4 insertions(+), 113 deletions(-) delete mode 100644 src/clj/protobuf/tasks.clj diff --git a/project.clj b/project.clj index 0575280..200576b 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,9 @@ -(defproject protobuf "0.5.0-alpha3" +(defproject protobuf "0.5.0-alpha4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] - :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.3"]] - :tasks [protobuf.tasks cake-marginalia.tasks] +;; :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.3"]] + :cake-plugins [[cake-protobuf "0.5.0-alpha4"]] +;; :tasks [cake-marginalia.tasks] :source-path ["src/clj" "src/jvm"] :jar-files ["proto"]) diff --git a/src/clj/protobuf/tasks.clj b/src/clj/protobuf/tasks.clj deleted file mode 100644 index 132b36c..0000000 --- a/src/clj/protobuf/tasks.clj +++ /dev/null @@ -1,110 +0,0 @@ -(ns protobuf.tasks - (:use cake cake.core uncle.core - [cake.file :only [file]] - [bake.core :only [log os-name]] - [cake.utils :only [prompt-read]] - [cake.tasks.compile :only [compile-java]] - [clojure.java.shell :only [sh]] - [clojure.java.io :only [reader]] - [bake.io :only [extract-resource]]) - (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar])) - -(def version "2.3.0") -(def srcdir (format "lib/protobuf-%s" version)) -(def tarfile (format "lib/protobuf-%s.tar.gz" version)) -(def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.tar.gz" version))) - -(defn installed? [] - (try (.contains (:out (sh "protoc" "--version")) version) - (catch java.io.IOException e))) - -(deftask fetch-protoc - (when-not (.exists (file srcdir)) - (ant Get {:src url :dest tarfile}) - (ant Untar {:src tarfile :dest "lib" :compression "gzip"}))) - -(deftask install-protoc - "Compile and install protoc to /usr/local." - (when-not (installed?) - (invoke fetch-protoc) - (when-not (.exists (file srcdir "src" "protoc")) - (ant Chmod {:file (file srcdir "configure") :perm "+x"}) - (ant Chmod {:file (file srcdir "install-sh") :perm "+x"}) - (ant ExecTask {:dir srcdir :executable "./configure"}) - (ant ExecTask {:dir srcdir :executable "make"})) - (let [password (prompt-read "Password" :echo false) - opts {:dir srcdir :input-string (str password "\n")}] - (if (= "linux" (os-name)) - (ant ExecTask (assoc opts :executable "script") - (argline "-q -c 'sudo -S make install' /dev/null")) - (ant ExecTask (assoc opts :executable "sudo") - (args ["-S" "make" "install"])))))) - -(deftask uninstall-protoc - "Remove protoc if it is installed." - (when (installed?) - (let [password (prompt-read "Password" :echo false)] - (ant ExecTask {:dir srcdir :executable "sudo" :input-string (str password "\n")} - (args ["-S" "make" "uninstall"]))))) - -(defn- proto-dependencies "look for lines starting with import in proto-file" - [proto-file] - (for [line (line-seq (reader proto-file)) :when (.startsWith line "import")] - (second (re-matches #".*\"(.*)\".*" line)))) - -(defn extract-dependencies "extract all files proto is dependent on" - [proto-file] - (loop [files (vec (proto-dependencies proto-file))] - (when-not (empty? files) - (let [proto (peek files) - files (pop files)] - (if (or (.exists (file "proto" proto)) (.exists (file "build/proto" proto))) - (recur files) - (let [proto-file (extract-resource (str "proto/" proto) "build")] - (recur (into files (proto-dependencies proto-file))))))))) - -(defn modtime [dir] - (let [files (rest (file-seq (file dir)))] - (if (empty? files) - 0 - (apply max (map #(.lastModified %) files))))) - -(defn protoc - ([protos] (protoc protos "build/protosrc")) - ([protos dest] - (when (or (:force *opts*) - (> (modtime "proto") (modtime dest)) - (> (modtime "proto") (modtime "classes"))) - (ant Mkdir {:dir dest}) - (ant Mkdir {:dir "build/proto"}) - (doseq [proto protos] - (log "Compiling" proto "to" dest) - (extract-dependencies (file "proto" proto)) - (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} - (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) - (catch org.apache.tools.ant.BuildException e - (throw (Exception. (str "error compiling " proto)))))) - (compile-java dest)))) - -(defn build-protobuf [] - (ant Mkdir {:dir "proto/google/protobuf"}) - (ant Copy {:file (str srcdir "/src/google/protobuf/descriptor.proto") :todir "proto/google/protobuf"}) - (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java")) - (protoc ["clojure/protobuf/extensions.proto" "clojure/protobuf/test.proto"])) - -(defn proto-file? [file] - (let [name (.getName file)] - (and (.endsWith name ".proto") - (not (.startsWith name "."))))) - -(defn proto-files [dir] - (for [file (rest (file-seq dir)) :when (proto-file? file)] - (.substring (.getPath file) (inc (count (.getPath dir)))))) - -(deftask compile-native #{proto}) -(deftask proto #{install-protoc} - "Compile protocol buffer files located in proto dir." - (if (= "protobuf" (:artifact-id *project*)) - (do (invoke fetch-protoc) - (build-protobuf)) - (protoc (or (:proto *opts*) (proto-files (file "proto")))))) From 901f273dc22d8002f4a9fded3ff67756dea11952 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 20 Sep 2011 21:45:58 -0700 Subject: [PATCH 143/525] bump versions --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 200576b..590d054 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,9 @@ -(defproject protobuf "0.5.0-alpha4" +(defproject protobuf "0.5.0-alpha5" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] ;; :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.3"]] - :cake-plugins [[cake-protobuf "0.5.0-alpha4"]] + :cake-plugins [[cake-protobuf "0.5.0-alpha5"]] ;; :tasks [cake-marginalia.tasks] :source-path ["src/clj" "src/jvm"] :jar-files ["proto"]) From 335a1dee9a372d0867e970b90a17d869683e8384 Mon Sep 17 00:00:00 2001 From: lance bradley Date: Wed, 21 Sep 2011 19:23:39 -0700 Subject: [PATCH 144/525] add load-stream --- project.clj | 2 +- src/clj/protobuf.clj | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 590d054..24ab2b8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-alpha5" +(defproject protobuf "0.5.0-alpha6" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index 3f2d33f..b709ccc 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -80,6 +80,12 @@ (let [^CodedInputStream in (CodedInputStream/newInstance data offset length)] (PersistentProtocolBufferMap/parseFrom type in))))) +(defn protobuf-load-stream + "Load a protobuf of the given type from an InputStream." + [^PersistentProtocolBufferMap$Def type ^InputStream stream] + (when stream + (let [^CodedInputStream in (CodedInputStream/newInstance stream)] + (PersistentProtocolBufferMap/parseFrom type in)))) (defn protobuf-dump "Return the byte representation of the given protobuf." From d0a5aad2ac27900263da7f1055981c5383c22071 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Sun, 25 Sep 2011 07:49:10 -0700 Subject: [PATCH 145/525] Update readme to mention cake-protobuf --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8e262ca..2c26154 100644 --- a/README.md +++ b/README.md @@ -94,11 +94,10 @@ Then you can access the maps in clojure: ## Installation -Add the `clojure-protobuf` to your [Cake](http://github.com/ninjudd/cake) project.clj: +Add `clojure-protobuf` and the `cake-protobuf` plugin to your [Cake](http://github.com/ninjudd/cake) project.clj: :dependencies [[clojure-protobuf "LATEST"]] - :dev-dependencies [[clojure-protobuf "LATEST"]] - :tasks [protobuf.tasks] - + :cake-plugins [[cake-protobuf "LATEST"]] + You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget to update this every time I bump the version, so I'm not going to put an actual number here. \ No newline at end of file From e0a4ef9e917861db82a9216dec8066cc2357a9d4 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 4 Oct 2011 16:17:15 -0700 Subject: [PATCH 146/525] Edited README.md via GitHub --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c26154..f1deaa4 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,11 @@ Then you can access the maps in clojure: ## Installation -Add `clojure-protobuf` and the `cake-protobuf` plugin to your [Cake](http://github.com/ninjudd/cake) project.clj: +As of cake 0.7.0, add `protobuf` and the `cake-protobuf` plugin to your [Cake](http://github.com/ninjudd/cake) project.clj: - :dependencies [[clojure-protobuf "LATEST"]] + :dependencies [[protobuf "LATEST"]] :cake-plugins [[cake-protobuf "LATEST"]] You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget -to update this every time I bump the version, so I'm not going to put an actual number here. \ No newline at end of file +to update this every time I bump the version, so I'm not going to put an actual number here. You can +find the lastest versions here (https://clojars.org/cake-protobuf https://clojars.org/protobuf). \ No newline at end of file From c9812a4604d9509ce0d8ec4095690c6992b8b922 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 4 Oct 2011 16:18:20 -0700 Subject: [PATCH 147/525] Edited README.md via GitHub --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f1deaa4..6e0cf95 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Then you can access the maps in clojure: ## Installation -As of cake 0.7.0, add `protobuf` and the `cake-protobuf` plugin to your [Cake](http://github.com/ninjudd/cake) project.clj: +As of [Cake](http://github.com/ninjudd/cake) 0.7.0, add `protobuf` and the `cake-protobuf` plugin to your project.clj: :dependencies [[protobuf "LATEST"]] :cake-plugins [[cake-protobuf "LATEST"]] From 778697e7fb1aaeae53c55aeb7b730832fe6713b8 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 7 Oct 2011 12:59:14 -0700 Subject: [PATCH 148/525] Edited README.md via GitHub --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e0cf95..f19e672 100644 --- a/README.md +++ b/README.md @@ -101,4 +101,6 @@ As of [Cake](http://github.com/ninjudd/cake) 0.7.0, add `protobuf` and the `cake You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget to update this every time I bump the version, so I'm not going to put an actual number here. You can -find the lastest versions here (https://clojars.org/cake-protobuf https://clojars.org/protobuf). \ No newline at end of file +find the lastest versions here (https://clojars.org/cake-protobuf https://clojars.org/protobuf). + +If you have any questions or need help, you can find us on IRC in [#flatland](irc://irc.freenode.net/#flatland). From e440ff2e6ca718cf2f557baac5f6ed37e9d7c49d Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 10 Oct 2011 16:38:46 -0700 Subject: [PATCH 149/525] remove adjoin --- src/clj/protobuf.clj | 5 ----- .../protobuf/PersistentProtocolBufferMap.java | 9 --------- test/protobuf_test.clj | 18 ------------------ 3 files changed, 32 deletions(-) diff --git a/src/clj/protobuf.clj b/src/clj/protobuf.clj index b709ccc..1607525 100644 --- a/src/clj/protobuf.clj +++ b/src/clj/protobuf.clj @@ -116,11 +116,6 @@ [^PersistentProtocolBufferMap p map] (.append p map)) -(defn adjoin - "Merge the given map into the protobuf. Like append, except default values in map will be used if it i." - [^PersistentProtocolBufferMap p map] - (.adjoin p map)) - (defn get-raw "Get value at key ignoring extension fields." [^PersistentProtocolBufferMap p key] diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java index a724aad..ba7389a 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java @@ -548,15 +548,6 @@ public PersistentProtocolBufferMap append(IPersistentMap map) { return new PersistentProtocolBufferMap(meta(), def, builder().mergeFrom(proto.message())); } - public PersistentProtocolBufferMap adjoin(IPersistentMap map) { - if (map instanceof PersistentProtocolBufferMap) { - PersistentProtocolBufferMap proto = (PersistentProtocolBufferMap) map; - return append(construct(def, proto.seq())); - } else { - return append(map); - } - } - public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) return this; diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 8b4dea4..77a6fb1 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -43,24 +43,6 @@ (is (= [1.2 3.4] (:doubles p))) (is (= [(float 0.01) (float 0.02)] (:floats p)))))) -(deftest test-adjoin - (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] - (let [p (adjoin p {:label "bar"})] - (is (= 5 (:id p))) - (is (= "bar" (:label p))) - (is (= false (:deleted p))) - (is (= ["little" "yellow"] (:tags p)))) - (let [p (adjoin (assoc p :deleted true) p)] - (is (= false (:deleted p)))) - (let [p (adjoin p {:tags ["different"]})] - (is (= ["little" "yellow" "different"] (:tags p)))) - (let [p (adjoin p {:tags ["different"] :label "very"})] - (is (= ["little" "yellow" "different"] (:tags p))) - (is (= "very" (:label p)))) - (let [p (adjoin p {:doubles [3.4] :floats [0.02]})] - (is (= [1.2 3.4] (:doubles p))) - (is (= [(float 0.01) (float 0.02)] (:floats p)))))) - (deftest test-assoc (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] (let [p (assoc p :label "baz" :tags ["nuprin"])] From c6b343315658768e336e058f6a24eed1ec53a9a6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 10 Oct 2011 16:57:09 -0700 Subject: [PATCH 150/525] move to protobuf.core --- README.md | 2 +- project.clj | 7 ++----- .../core}/extensions.proto | 2 +- .../protobuf => protobuf/core}/test.proto | 4 ++-- src/jvm/.gitignore | 1 - src/{clj/protobuf.clj => protobuf/core.clj} | 4 ++-- .../core}/PersistentProtocolBufferMap.java | 2 +- tasks.clj | 14 ++++++++++++++ test/protobuf_test.clj | 18 +++++++++--------- 9 files changed, 32 insertions(+), 22 deletions(-) rename proto/{clojure/protobuf => protobuf/core}/extensions.proto (93%) rename proto/{clojure/protobuf => protobuf/core}/test.proto (96%) delete mode 100644 src/jvm/.gitignore rename src/{clj/protobuf.clj => protobuf/core.clj} (97%) rename src/{jvm/clojure/protobuf => protobuf/core}/PersistentProtocolBufferMap.java (99%) create mode 100644 tasks.clj diff --git a/README.md b/README.md index f19e672..070cbfd 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ As of [Cake](http://github.com/ninjudd/cake) 0.7.0, add `protobuf` and the `cake :dependencies [[protobuf "LATEST"]] :cake-plugins [[cake-protobuf "LATEST"]] - + You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget to update this every time I bump the version, so I'm not going to put an actual number here. You can find the lastest versions here (https://clojars.org/cake-protobuf https://clojars.org/protobuf). diff --git a/project.clj b/project.clj index 24ab2b8..812adbb 100644 --- a/project.clj +++ b/project.clj @@ -1,9 +1,6 @@ -(defproject protobuf "0.5.0-alpha6" +(defproject protobuf "0.5.0-beta1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] -;; :dev-dependencies [[org.clojars.flatland/cake-marginalia "0.6.3"]] - :cake-plugins [[cake-protobuf "0.5.0-alpha5"]] -;; :tasks [cake-marginalia.tasks] - :source-path ["src/clj" "src/jvm"] + :cake-plugins [[cake-protobuf "0.5.0-beta1"]] :jar-files ["proto"]) diff --git a/proto/clojure/protobuf/extensions.proto b/proto/protobuf/core/extensions.proto similarity index 93% rename from proto/clojure/protobuf/extensions.proto rename to proto/protobuf/core/extensions.proto index f72066e..9d61025 100644 --- a/proto/clojure/protobuf/extensions.proto +++ b/proto/protobuf/core/extensions.proto @@ -1,6 +1,6 @@ import "google/protobuf/descriptor.proto"; -option java_package = "clojure.protobuf"; +option java_package = "protobuf.core"; option java_outer_classname = "Extensions"; extend google.protobuf.FieldOptions { diff --git a/proto/clojure/protobuf/test.proto b/proto/protobuf/core/test.proto similarity index 96% rename from proto/clojure/protobuf/test.proto rename to proto/protobuf/core/test.proto index 7149e8d..df6e255 100644 --- a/proto/clojure/protobuf/test.proto +++ b/proto/protobuf/core/test.proto @@ -1,6 +1,6 @@ -import "clojure/protobuf/extensions.proto"; +import "protobuf/core/extensions.proto"; -option java_package = "clojure.protobuf"; +option java_package = "protobuf.core"; option java_outer_classname = "Test"; message Foo { diff --git a/src/jvm/.gitignore b/src/jvm/.gitignore deleted file mode 100644 index e2cb692..0000000 --- a/src/jvm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Collections.java \ No newline at end of file diff --git a/src/clj/protobuf.clj b/src/protobuf/core.clj similarity index 97% rename from src/clj/protobuf.clj rename to src/protobuf/core.clj index 1607525..2a2ef37 100644 --- a/src/clj/protobuf.clj +++ b/src/protobuf/core.clj @@ -1,7 +1,7 @@ -(ns protobuf +(ns protobuf.core (:use [clojure.java.io :only [input-stream output-stream]] [clojure.string :only [lower-case]]) - (:import (clojure.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) + (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream) (java.io InputStream OutputStream))) diff --git a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java similarity index 99% rename from src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java rename to src/protobuf/core/PersistentProtocolBufferMap.java index ba7389a..3cc9a4b 100644 --- a/src/jvm/clojure/protobuf/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -8,7 +8,7 @@ * You must not remove this notice, or any other, from this software. **/ -package clojure.protobuf; +package protobuf.core; import clojure.lang.*; import java.util.*; diff --git a/tasks.clj b/tasks.clj new file mode 100644 index 0000000..5b37b4e --- /dev/null +++ b/tasks.clj @@ -0,0 +1,14 @@ +(ns tasks + (:use [cake.core :only [deftask]] + [cake.file :only [file cp mkdir]] + [cake.tasks.protobuf :only [protoc srcdir]])) + +(defn build-protobuf [] + (protoc ["protobuf/core/extensions.proto" "clojure/protobuf/test.proto"])) + +(deftask protobuf.google + (mkdir "proto/google/protobuf") + (cp (str srcdir "/src/google/protobuf/descriptor.proto") "proto/google/protobuf") + (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) + +(deftask proto #{protobuf.google}) diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index 77a6fb1..ffbf02a 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -1,12 +1,12 @@ (ns protobuf-test - (:use protobuf) + (:use protobuf.core) (:use clojure.test) (:import (java.io PipedInputStream PipedOutputStream))) -(defprotobuf Foo clojure.protobuf.Test Foo) -(defprotobuf Bar clojure.protobuf.Test Bar) -(defprotobuf Response clojure.protobuf.Test Response) -(defprotobuf ErrorMsg clojure.protobuf.Test ErrorMsg) +(defprotobuf Foo protobuf.core.Test Foo) +(defprotobuf Bar protobuf.core.Test Bar) +(defprotobuf Response protobuf.core.Test Response) +(defprotobuf ErrorMsg protobuf.core.Test ErrorMsg) (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) @@ -208,7 +208,7 @@ :long {:type :float}, :deleted {:type :boolean}}] (is (= fields (protofields Foo))) - (is (= fields (protofields clojure.protobuf.Test$Foo))))) + (is (= fields (protofields protobuf.core.Test$Foo))))) (deftest test-nested-protofields (is (= {:year {:type :int}, @@ -237,14 +237,14 @@ (is (= {} (protodefault Foo :groups))) (is (= {} (protodefault Foo :item-map))) (is (= false (protodefault Foo :deleted))) - (is (= {} (protodefault clojure.protobuf.Test$Foo :groups)))) + (is (= {} (protodefault protobuf.core.Test$Foo :groups)))) (deftest test-use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] (is (= '(:id :responses :tag-set :deleted) (keys p))) (is (= [:yes :not-sure :maybe :not-sure :no] (:responses p))) - (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores true) + (protobuf.core.PersistentProtocolBufferMap/setUseUnderscores true) (is (= '(:id :responses :tag_set :deleted) (keys p))) (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) @@ -252,7 +252,7 @@ :pair_map :groups :doubles :floats :item_map :counts :time :lat :long} (set (keys (protofields Foo))))) - (clojure.protobuf.PersistentProtocolBufferMap/setUseUnderscores false))) + (protobuf.core.PersistentProtocolBufferMap/setUseUnderscores false))) (deftest test-protobuf-nested-message (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))] From 8f51eed3b72f48991024f21588e02886a08a54b2 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 21 Oct 2011 16:12:37 -0700 Subject: [PATCH 151/525] fix reflection warnings --- src/protobuf/core.clj | 28 +++++++++++-------- .../core/PersistentProtocolBufferMap.java | 10 ------- test/protobuf_test.clj | 11 ++++---- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 2a2ef37..6b80841 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -2,8 +2,10 @@ (:use [clojure.java.io :only [input-stream output-stream]] [clojure.string :only [lower-case]]) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) - (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor CodedInputStream) - (java.io InputStream OutputStream))) + (com.google.protobuf GeneratedMessage CodedInputStream + Descriptors$Descriptor Descriptors$FieldDescriptor) + (java.io InputStream OutputStream) + (clojure.lang Reflector))) (defn protobuf? "Is the given object a PersistentProtocolBufferMap?" @@ -17,16 +19,20 @@ (defn protodef "Create a protodef from a string or protobuf class." - ([class] - (if (or (protodef? class) (nil? class)) - class - (PersistentProtocolBufferMap$Def/create class))) - ([class & fields] - (loop [type (protodef class) + ([def] + (if (or (protodef? def) (nil? def)) + def + (PersistentProtocolBufferMap$Def/create + ^Descriptors.Descriptor + (if (instance? Descriptors$Descriptor def) + def + (Reflector/invokeStaticMethod ^Class def "getDescriptor" (to-array nil)))))) + ([def & fields] + (loop [^PersistentProtocolBufferMap$Def def (protodef def) fields fields] (if (empty? fields) - type - (recur (-> type + def + (recur (-> def (.fieldDescriptor (first fields)) .getMessageType protodef) @@ -35,7 +41,7 @@ (defmacro defprotobuf "Helper macro for defining a protodef object." [sym & args] - (let [class (apply str (interpose "$" (map name args)))] + (let [class (symbol (apply str (interpose "$" (map name args))))] `(def ~sym (protodef ~class)))) (defn protobuf diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 3cc9a4b..65326e6 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -32,16 +32,6 @@ public static class Def { ConcurrentHashMap keyword_to_field; static ConcurrentHashMap type_to_def = new ConcurrentHashMap(); - public static Def create(String class_name) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, ClassNotFoundException { - Class c = Class.forName(class_name); - return create(c); - } - - public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); - return create(type); - } - public static Def create(Descriptors.Descriptor type) { Def def = type_to_def.get(type); if (def == null) { diff --git a/test/protobuf_test.clj b/test/protobuf_test.clj index ffbf02a..38de66e 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf_test.clj @@ -1,12 +1,11 @@ (ns protobuf-test - (:use protobuf.core) - (:use clojure.test) + (:use protobuf.core clojure.test) (:import (java.io PipedInputStream PipedOutputStream))) -(defprotobuf Foo protobuf.core.Test Foo) -(defprotobuf Bar protobuf.core.Test Bar) -(defprotobuf Response protobuf.core.Test Response) -(defprotobuf ErrorMsg protobuf.core.Test ErrorMsg) +(def Foo (protodef protobuf.core.Test$Foo)) +(def Bar (protodef protobuf.core.Test$Bar)) +(def Response (protodef protobuf.core.Test$Response)) +(def ErrorMsg (protodef protobuf.core.Test$ErrorMsg)) (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) From 38e2fef4ead0b738cd7bb6763d6b26da8462fccb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 21 Oct 2011 16:17:50 -0700 Subject: [PATCH 152/525] beta2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 812adbb..38607fe 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-beta1" +(defproject protobuf "0.5.0-beta2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] From 45f12c87ecfa29811f0a891721ba41dbbfec3521 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 28 Oct 2011 18:26:36 -0700 Subject: [PATCH 153/525] add gloss support --- project.clj | 4 +- proto/protobuf/test/codec.proto | 44 +++++++++++++++++ .../{core/test.proto => test/core.proto} | 4 +- src/protobuf/codec.clj | 24 ++++++++++ src/protobuf/core.clj | 6 --- .../core/PersistentProtocolBufferMap.java | 5 ++ test/protobuf/codec_test.clj | 47 +++++++++++++++++++ .../core_test.clj} | 10 ++-- 8 files changed, 130 insertions(+), 14 deletions(-) create mode 100644 proto/protobuf/test/codec.proto rename proto/protobuf/{core/test.proto => test/core.proto} (96%) create mode 100644 src/protobuf/codec.clj create mode 100644 test/protobuf/codec_test.clj rename test/{protobuf_test.clj => protobuf/core_test.clj} (98%) diff --git a/project.clj b/project.clj index 38607fe..b749b8a 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,8 @@ -(defproject protobuf "0.5.0-beta2" +(defproject protobuf "0.5.0-beta3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"]] + :test-dependencies [[gloss "0.2.0-rc1"] + [io "0.1.0-alpha1"]] :cake-plugins [[cake-protobuf "0.5.0-beta1"]] :jar-files ["proto"]) diff --git a/proto/protobuf/test/codec.proto b/proto/protobuf/test/codec.proto new file mode 100644 index 0000000..7c4a911 --- /dev/null +++ b/proto/protobuf/test/codec.proto @@ -0,0 +1,44 @@ +import "protobuf/core/extensions.proto"; + +option java_package = "protobuf.test"; +option java_outer_classname = "Codec"; + +message Foo { + optional int32 foo = 1; + optional int32 bar = 2; + optional int32 baz = 3; + + repeated string tags = 4; + repeated Item tag_set = 5 [(set) = true]; + repeated Entry num_map = 6 [(map) = true]; + + optional Foo nested = 7; + + optional int32 rev = 8; +} + +message Item { + required string item = 1; + required bool exists = 2 [default = true]; +} + +message Entry { + required int32 key = 1; + required string val = 2; +} + +message Edge { + required string to_id = 1; + optional string a = 2; + optional string b = 3; + optional bool deleted = 4; +} + +message Node { + optional string id = 1; + repeated Edge edges = 2 [(map_by) = "to_id"]; + optional int32 rev = 3; + optional int32 foo = 4; + optional string bar = 5; + repeated int32 baz = 6; +} \ No newline at end of file diff --git a/proto/protobuf/core/test.proto b/proto/protobuf/test/core.proto similarity index 96% rename from proto/protobuf/core/test.proto rename to proto/protobuf/test/core.proto index df6e255..8b8040a 100644 --- a/proto/protobuf/core/test.proto +++ b/proto/protobuf/test/core.proto @@ -1,7 +1,7 @@ import "protobuf/core/extensions.proto"; -option java_package = "protobuf.core"; -option java_outer_classname = "Test"; +option java_package = "protobuf.test"; +option java_outer_classname = "Core"; message Foo { optional uint32 id = 1 [default = 43]; diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj new file mode 100644 index 0000000..dd55025 --- /dev/null +++ b/src/protobuf/codec.clj @@ -0,0 +1,24 @@ +(ns protobuf.codec + (:use protobuf.core + [gloss.core.protocols :only [Reader Writer]] + [gloss.core.formats :only [to-buf-seq]] + [clojure.java.io :only [input-stream]]) + (:require io.core)) + +(defn protobuf-codec [proto & {:keys [validator]}] + (let [proto (protodef proto)] + (reify + Reader + (read-bytes [this buf-seq] + [true (protobuf-load-stream proto (input-stream buf-seq)) nil]) + + Writer + (sizeof [this] nil) + (write-bytes [this _ val] + (when (and validator (not (validator val))) + (throw (IllegalStateException. "Invalid value in protobuf-codec"))) + (to-buf-seq + (protobuf-dump + (if (protobuf? val) + val + (protobuf proto val)))))))) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 6b80841..546c061 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -38,12 +38,6 @@ protodef) (rest fields)))))) -(defmacro defprotobuf - "Helper macro for defining a protodef object." - [sym & args] - (let [class (symbol (apply str (interpose "$" (map name args))))] - `(def ~sym (protodef ~class)))) - (defn protobuf "Construct a protobuf of the given type." ([^PersistentProtocolBufferMap$Def type] diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 65326e6..3fc0ac3 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -32,6 +32,11 @@ public static class Def { ConcurrentHashMap keyword_to_field; static ConcurrentHashMap type_to_def = new ConcurrentHashMap(); + public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); + return create(type); + } + public static Def create(Descriptors.Descriptor type) { Def def = type_to_def.get(type); if (def == null) { diff --git a/test/protobuf/codec_test.clj b/test/protobuf/codec_test.clj new file mode 100644 index 0000000..c5adfd3 --- /dev/null +++ b/test/protobuf/codec_test.clj @@ -0,0 +1,47 @@ +(ns protobuf.codec-test + (:use clojure.test gloss.io cereal.core protobuf.codec) + (:import (java.nio ByteBuffer))) + +(deftest protobuf-codec-test + (let [codec (protobuf-codec protobuf.test.Codec$Foo)] + (testing "decode an encoded data structure" + (let [val {:foo 1 :bar 2}] + (is (= val (decode codec (encode codec val)))))) + + (testing "append two simple encoded data structures" + (let [data1 (encode codec {:foo 1 :bar 2}) + data2 (encode codec {:foo 4 :baz 8})] + (is (= {:foo 4 :bar 2 :baz 8} + (decode codec (concat data1 data2)))))) + + (testing "concat lists when appending" + (let [data1 (encode codec {:tags ["foo" "bar"] :foo 1}) + data2 (encode codec {:tags ["baz" "foo"] :foo 2})] + (is (= {:foo 2 :tags ["foo" "bar" "baz" "foo"]} + (decode codec (concat data1 data2)))))) + + (testing "merge maps when appending" + (let [data1 (encode codec {:num-map {1 "one" 3 "three"}}) + data2 (encode codec {:num-map {2 "dos" 3 "tres"}}) + data3 (encode codec {:num-map {3 "san" 6 "roku"}})] + (is (= {:num-map {1 "one" 2 "dos" 3 "san" 6 "roku"}} + (decode codec (concat data1 data2 data3)))))) + + (testing "merge sets when appending" + (let [data1 (encode codec {:tag-set #{"foo" "bar"}}) + data2 (encode codec {:tag-set #{"baz" "foo"}})] + (is (= {:tag-set #{"foo" "bar" "baz"}} + (decode codec (concat data1 data2)))))) + + (testing "support set deletion using existence map" + (let [data1 (encode codec {:tag-set #{"foo" "bar" "baz"}}) + data2 (encode codec {:tag-set {"baz" false "foo" true "zap" true "bam" false}})] + (is (= {:tag-set #{"foo" "bar" "zap"}} + (decode codec (concat data1 data2)))))) + + (testing "merge and append nested data structures when appending" + (let [data1 (encode codec {:nested {:foo 1 :tags ["bar"] :nested {:tag-set #{"a" "c"}}}}) + data2 (encode codec {:nested {:foo 4 :tags ["baz"] :bar 3}}) + data3 (encode codec {:nested {:baz 5 :tags ["foo"] :nested {:tag-set {"b" true "c" false}}}})] + (is (= {:nested {:foo 4 :bar 3 :baz 5 :tags ["bar" "baz" "foo"] :nested {:tag-set #{"a" "b"}}}} + (decode codec (concat data1 data2 data3)))))))) diff --git a/test/protobuf_test.clj b/test/protobuf/core_test.clj similarity index 98% rename from test/protobuf_test.clj rename to test/protobuf/core_test.clj index 38de66e..6b60a83 100644 --- a/test/protobuf_test.clj +++ b/test/protobuf/core_test.clj @@ -1,11 +1,11 @@ -(ns protobuf-test +(ns protobuf.core-test (:use protobuf.core clojure.test) (:import (java.io PipedInputStream PipedOutputStream))) -(def Foo (protodef protobuf.core.Test$Foo)) -(def Bar (protodef protobuf.core.Test$Bar)) -(def Response (protodef protobuf.core.Test$Response)) -(def ErrorMsg (protodef protobuf.core.Test$ErrorMsg)) +(def Foo (protodef protobuf.test.Core$Foo)) +(def Bar (protodef protobuf.test.Core$Bar)) +(def Response (protodef protobuf.test.Core$Response)) +(def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) (defn catbytes [& args] (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) From c9a8631bfbfe37b63ccb6a9e8a437e8e5acf3a6e Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 31 Oct 2011 14:40:51 -0700 Subject: [PATCH 154/525] add support for repeated protobuf-codec using :_len prefix --- project.clj | 3 +- proto/protobuf/test/codec.proto | 3 +- src/protobuf/codec.clj | 49 +++++++++++++++++++++------------ src/protobuf/core.clj | 6 ++-- test/protobuf/codec_test.clj | 14 +++++++++- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/project.clj b/project.clj index b749b8a..2b90d0f 100644 --- a/project.clj +++ b/project.clj @@ -1,7 +1,8 @@ (defproject protobuf "0.5.0-beta3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] - [ordered-set "0.2.2"]] + [ordered-set "0.2.2"] + [useful "0.7.4-alpha4"]] :test-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha1"]] :cake-plugins [[cake-protobuf "0.5.0-beta1"]] diff --git a/proto/protobuf/test/codec.proto b/proto/protobuf/test/codec.proto index 7c4a911..02b1a65 100644 --- a/proto/protobuf/test/codec.proto +++ b/proto/protobuf/test/codec.proto @@ -14,7 +14,8 @@ message Foo { optional Foo nested = 7; - optional int32 rev = 8; + repeated int32 revisions = 8; + optional fixed32 _len = 9; } message Item { diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index dd55025..1b15a24 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -2,23 +2,38 @@ (:use protobuf.core [gloss.core.protocols :only [Reader Writer]] [gloss.core.formats :only [to-buf-seq]] + [useful.fn :only [fix]] [clojure.java.io :only [input-stream]]) - (:require io.core)) + (:require io.core + [gloss.core :as gloss])) -(defn protobuf-codec [proto & {:keys [validator]}] - (let [proto (protodef proto)] - (reify - Reader - (read-bytes [this buf-seq] - [true (protobuf-load-stream proto (input-stream buf-seq)) nil]) +(declare protobuf-codec) + +(defn length-prefix [proto] + (let [proto (protodef proto) + min (alength (protobuf-dump proto {:_len 0})) + max (alength (protobuf-dump proto {:_len Integer/MAX_VALUE}))] + (when-not (= min max) + (throw (Exception. "_len must be of type fixed32 or fixed64"))) + (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) + #(hash-map :_len %) + :_len))) - Writer - (sizeof [this] nil) - (write-bytes [this _ val] - (when (and validator (not (validator val))) - (throw (IllegalStateException. "Invalid value in protobuf-codec"))) - (to-buf-seq - (protobuf-dump - (if (protobuf? val) - val - (protobuf proto val)))))))) +(defn protobuf-codec [proto & {:keys [validator repeated]}] + (let [proto (protodef proto)] + (-> (reify + Reader + (read-bytes [this buf-seq] + [true (protobuf-load-stream proto (input-stream buf-seq)) nil]) + Writer + (sizeof [this] nil) + (write-bytes [this _ val] + (when (and validator (not (validator val))) + (throw (IllegalStateException. "Invalid value in protobuf-codec"))) + (to-buf-seq + (protobuf-dump + (if (protobuf? val) + val + (protobuf proto val)))))) + (fix repeated + #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none))))) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 546c061..3242fd3 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -89,8 +89,10 @@ (defn protobuf-dump "Return the byte representation of the given protobuf." - [^PersistentProtocolBufferMap p] - (.toByteArray p)) + ([^PersistentProtocolBufferMap p] + (.toByteArray p)) + ([^PersistentProtocolBufferMap$Def type m] + (protobuf-dump (PersistentProtocolBufferMap/construct type m)))) (defn protobuf-seq "Lazily read a sequence of length-delimited protobufs of the specified type from the given input stream." diff --git a/test/protobuf/codec_test.clj b/test/protobuf/codec_test.clj index c5adfd3..4c062c9 100644 --- a/test/protobuf/codec_test.clj +++ b/test/protobuf/codec_test.clj @@ -1,5 +1,5 @@ (ns protobuf.codec-test - (:use clojure.test gloss.io cereal.core protobuf.codec) + (:use clojure.test gloss.io protobuf.codec) (:import (java.nio ByteBuffer))) (deftest protobuf-codec-test @@ -45,3 +45,15 @@ data3 (encode codec {:nested {:baz 5 :tags ["foo"] :nested {:tag-set {"b" true "c" false}}}})] (is (= {:nested {:foo 4 :bar 3 :baz 5 :tags ["bar" "baz" "foo"] :nested {:tag-set #{"a" "b"}}}} (decode codec (concat data1 data2 data3)))))))) + +(deftest repeated-protobufs + (let [len (length-prefix protobuf.test.Codec$Foo) + codec (protobuf-codec protobuf.test.Codec$Foo :repeated true)] + (testing "length-prefix" + (doseq [i [0 10 100 1000 10000 100000 Integer/MAX_VALUE]] + (is (= i (decode len (encode len i)))))) + (testing "repeated" + (let [data1 (encode codec [{:foo 1 :bar 2}]) + data2 (encode codec [{:foo 4 :baz 8}])] + (is (= [{:foo 1 :bar 2} {:foo 4 :baz 8}] + (decode codec (concat data1 data2)))))))) \ No newline at end of file From 32d8cd832cd3561f6b7aace23240d3f477008c94 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 31 Oct 2011 14:41:48 -0700 Subject: [PATCH 155/525] beta4 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2b90d0f..4f8271b 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-beta3" +(defproject protobuf "0.5.0-beta4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] From 3e98aa5068919bcaaf8c68b1dcd022ea04b09ac7 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 22 Nov 2011 16:02:23 -0600 Subject: [PATCH 156/525] WIP on getting things on Leiningen. Also got rid of a bunch of ant tasks. --- project.clj | 10 ++- src/leiningen/protobuf.clj | 153 +++++++++++++++++++++++++++++++++++++ tasks.clj | 14 ---- 3 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 src/leiningen/protobuf.clj delete mode 100644 tasks.clj diff --git a/project.clj b/project.clj index 4f8271b..8adb113 100644 --- a/project.clj +++ b/project.clj @@ -2,8 +2,10 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] - [useful "0.7.4-alpha4"]] - :test-dependencies [[gloss "0.2.0-rc1"] - [io "0.1.0-alpha1"]] - :cake-plugins [[cake-protobuf "0.5.0-beta1"]] + [useful "0.7.4-alpha4"] + [uncle "0.1.0"] + [classlojure "0.6.3"]] + :dev-dependencies [[gloss "0.2.0-rc1"] + [io "0.1.0-alpha1"]] + :eval-in-leiningen true :jar-files ["proto"]) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj new file mode 100644 index 0000000..01b1572 --- /dev/null +++ b/src/leiningen/protobuf.clj @@ -0,0 +1,153 @@ +(ns leiningen.protobuf + (:use [clojure.java.shell :only [sh]] + [clojure.string :only [join]] + [leiningen.help :only [help-for]] + [leiningen.javac :only [javac]] + [leiningen.util.paths :only [get-os]] + [leiningen.core :only [prepend-tasks]] + uncle.core) + (:require [clojure.java.io :as io]) + (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar] + java.util.zip.ZipFile)) + +(def version "2.3.0") +(def srcdir (format "lib/protobuf-%s" version)) +(def zipfile (format "lib/protobuf-%s.zip" version)) +(def url (java.net.URL. + (format + "http://protobuf.googlecode.com/files/protobuf-%s.zip" + version))) + +(defn- unzip [source target-dir] + (let [zip (ZipFile. source) + entries (enumeration-seq (.entries zip)) + target-file #(io/file target-dir (.getName %))] + (doseq [entry entries :when (not (.isDirectory entry)) + :let [f (target-file entry)]] + (.mkdirs (.getParentFile f)) + (io/copy (.getInputStream zip entry) f)))) + +(defn- proto-dependencies "look for lines starting with import in proto-file" + [proto-file] + (for [line (line-seq (io/reader proto-file)) :when (.startsWith line "import")] + (second (re-matches #".*\"(.*)\".*" line)))) + +(defn extract-dependencies "extract all files proto is dependent on" + [proto-file] + (loop [files (vec (proto-dependencies proto-file))] + (when-not (empty? files) + (let [proto (peek files) + files (pop files)] + (if (or (.exists (io/file "proto" proto)) (.exists (io/file "build/proto" proto))) + (recur files) + (let [location (str "proto/" proto) + proto-file (io/file "build" location)] + (io/copy (io/reader (io/resource location)) proto-file) + (recur (into files (proto-dependencies proto-file))))))))) + +(defn modtime [dir] + (let [files (rest (file-seq (io/file dir)))] + (if (empty? files) + 0 + (apply max (map #(.lastModified %) files))))) + +(defn proto-file? [file] + (let [name (.getName file)] + (and (.endsWith name ".proto") + (not (.startsWith name "."))))) + +(defn proto-files [dir] + (for [file (rest (file-seq dir)) :when (proto-file? file)] + (.substring (.getPath file) (inc (count (.getPath dir)))))) + +(defn installed? [] + (try (.contains (:out (sh "protoc" "--version")) version) + (catch java.io.IOException e))) + +(defn read-pass [] + (print "Password: ") + (flush) + (join (.readPassword (System/console)))) + +(defn fetch [project] + (when-not (.exists (io/file srcdir)) + (with-open [stream (.openStream url)] + (io/copy stream (io/file zipfile))) + (unzip (io/file zipfile) "lib"))) + +(defn uninstall + "Remove protoc if it is installed." + [project] + (when (installed?) + (let [password (read-pass)] + (sh "sudo" "-S" "make" "uninstall" + :dir srcdir :in (str password "\n"))))) + +(defn install + "Compile and install protoc to /usr/local." + [project] + (when-not (installed?) + (fetch nil) + (when-not (.exists (io/file srcdir "src" "protoc")) + (.setExecutable (io/file srcdir "configure") true) + (.setExecutable (io/file srcdir "install-sh") true) + (println "Configuring protoc...") + (sh "./configure" :dir srcdir) + (println "Running 'make'...") + (sh "make" :dir srcdir)) + (let [password (str (read-pass) "\n") + opts {:dir srcdir :input-string (str password "\n")}] + (println "Installing...") + (if (= :linux (get-os)) + (sh "script" "-q" "-c" "sudo -S make install" "/dev/null" + :dir srcdir :in password) + (sh "sudo" "-S" "make" "install" + :dir srcdir :in password))))) + +(defn protoc + ([project protos] (protoc project "build/protosrc" protos)) + ([project dest protos] + (when (or (> (modtime "proto") (modtime dest)) + (> (modtime "proto") (modtime "classes"))) + (.mkdirs (io/file dest)) + (.mkdir (io/file "build" "proto")) + (doseq [proto protos] + (println "Compiling" proto "to" dest) + (extract-dependencies (io/file "proto" proto)) + (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} + (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) + (catch org.apache.tools.ant.BuildException e + (throw (Exception. (str "error compiling " proto)))))) + (javac project dest)))) + +(defn compile + "Compile protocol buffer files located in proto dir." + ([project] (compile project (proto-files (io/file "proto")))) + ([project files] + (install) + (protoc project files))) + +(defn google + [project] + (let [proto-files (io/file "proto/google/protobuf")] + (.mkdirs proto-files) + (io/copy + (io/reader + (str srcdir "/src/google/protobuf/descriptor.proto")) + proto-files)) + (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) + +(prepend-tasks #'javac compile) + +(defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." + :help-arglists '([subtask & args]) + :subtasks [#'fetch #'install #'uninstall #'compile]} + protobuf + ([project] (println (help-for "protobuf"))) + ([project subtask & args] + (case subtask + "proto" (apply google project args) + "fetch" (apply fetch project args) + "install" (apply install project args) + "uninstall" (apply uninstall project args) + "compile" (apply compile project args)))) \ No newline at end of file diff --git a/tasks.clj b/tasks.clj deleted file mode 100644 index 5b37b4e..0000000 --- a/tasks.clj +++ /dev/null @@ -1,14 +0,0 @@ -(ns tasks - (:use [cake.core :only [deftask]] - [cake.file :only [file cp mkdir]] - [cake.tasks.protobuf :only [protoc srcdir]])) - -(defn build-protobuf [] - (protoc ["protobuf/core/extensions.proto" "clojure/protobuf/test.proto"])) - -(deftask protobuf.google - (mkdir "proto/google/protobuf") - (cp (str srcdir "/src/google/protobuf/descriptor.proto") "proto/google/protobuf") - (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) - -(deftask proto #{protobuf.google}) From 7b052df698e0a9819f9bf9cf362ba0e126cc5a75 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 22 Nov 2011 17:09:06 -0600 Subject: [PATCH 157/525] Newlines are cool. --- src/leiningen/protobuf.clj | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 01b1572..154de86 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -38,7 +38,8 @@ (when-not (empty? files) (let [proto (peek files) files (pop files)] - (if (or (.exists (io/file "proto" proto)) (.exists (io/file "build/proto" proto))) + (if (or (.exists (io/file "proto" proto)) + (.exists (io/file "build/proto" proto))) (recur files) (let [location (str "proto/" proto) proto-file (io/file "build" location)] @@ -69,7 +70,7 @@ (flush) (join (.readPassword (System/console)))) -(defn fetch [project] +(defn fetch [] (when-not (.exists (io/file srcdir)) (with-open [stream (.openStream url)] (io/copy stream (io/file zipfile))) @@ -77,7 +78,7 @@ (defn uninstall "Remove protoc if it is installed." - [project] + [] (when (installed?) (let [password (read-pass)] (sh "sudo" "-S" "make" "uninstall" @@ -85,9 +86,9 @@ (defn install "Compile and install protoc to /usr/local." - [project] + [] (when-not (installed?) - (fetch nil) + (fetch) (when-not (.exists (io/file srcdir "src" "protoc")) (.setExecutable (io/file srcdir "configure") true) (.setExecutable (io/file srcdir "install-sh") true) @@ -128,7 +129,7 @@ (protoc project files))) (defn google - [project] + [] (let [proto-files (io/file "proto/google/protobuf")] (.mkdirs proto-files) (io/copy @@ -146,8 +147,8 @@ ([project] (println (help-for "protobuf"))) ([project subtask & args] (case subtask - "proto" (apply google project args) - "fetch" (apply fetch project args) - "install" (apply install project args) - "uninstall" (apply uninstall project args) - "compile" (apply compile project args)))) \ No newline at end of file + "proto" (apply google args) + "fetch" (apply fetch args) + "install" (apply install args) + "uninstall" (apply uninstall args) + "compile" (apply compile args)))) \ No newline at end of file From 895d3df25b63b7f6ebf38ed0ea0bf17e88770a4d Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 22 Nov 2011 17:19:39 -0600 Subject: [PATCH 158/525] eval-in-project. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 8adb113..f148802 100644 --- a/project.clj +++ b/project.clj @@ -7,5 +7,5 @@ [classlojure "0.6.3"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha1"]] - :eval-in-leiningen true + :eval-in-project true :jar-files ["proto"]) From e18fa4f25565474fb77388ec612fad8cc456d3a5 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 23 Nov 2011 15:42:38 -0600 Subject: [PATCH 159/525] Almost done, slightly broken. --- project.clj | 2 +- src/leiningen/protobuf.clj | 38 +++++++++++++++----------------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/project.clj b/project.clj index f148802..5ab8627 100644 --- a/project.clj +++ b/project.clj @@ -3,9 +3,9 @@ :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] [useful "0.7.4-alpha4"] - [uncle "0.1.0"] [classlojure "0.6.3"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha1"]] :eval-in-project true + :java-source-path "build/protosrc" :jar-files ["proto"]) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 154de86..b18d74b 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -4,11 +4,9 @@ [leiningen.help :only [help-for]] [leiningen.javac :only [javac]] [leiningen.util.paths :only [get-os]] - [leiningen.core :only [prepend-tasks]] - uncle.core) + [leiningen.core :only [prepend-tasks]]) (:require [clojure.java.io :as io]) - (:import [org.apache.tools.ant.taskdefs Chmod Copy ExecTask Get Javac Mkdir Untar] - java.util.zip.ZipFile)) + (:import java.util.zip.ZipFile)) (def version "2.3.0") (def srcdir (format "lib/protobuf-%s" version)) @@ -106,8 +104,8 @@ :dir srcdir :in password))))) (defn protoc - ([project protos] (protoc project "build/protosrc" protos)) - ([project dest protos] + ([project protos] (protoc project protos "build/protosrc")) + ([project protos dest] (when (or (> (modtime "proto") (modtime dest)) (> (modtime "proto") (modtime "classes"))) (.mkdirs (io/file dest)) @@ -115,30 +113,24 @@ (doseq [proto protos] (println "Compiling" proto "to" dest) (extract-dependencies (io/file "proto" proto)) - (try (ant ExecTask {:executable "protoc" :dir "proto" :failonerror true} - (args [proto (str "--java_out=../" dest) "-I." "-I../build/proto"])) - (catch org.apache.tools.ant.BuildException e - (throw (Exception. (str "error compiling " proto)))))) - (javac project dest)))) + (sh "protoc" proto (str "--java_out=../" dest) "-I." "-I../build/proto" + :dir "proto")) + (javac (assoc project :java-source-path dest))))) -(defn compile +(defn compile-protos "Compile protocol buffer files located in proto dir." - ([project] (compile project (proto-files (io/file "proto")))) + ([project] (compile-protos project (proto-files (io/file "proto")))) ([project files] (install) (protoc project files))) (defn google - [] + [project] (let [proto-files (io/file "proto/google/protobuf")] (.mkdirs proto-files) - (io/copy - (io/reader - (str srcdir "/src/google/protobuf/descriptor.proto")) - proto-files)) - (protoc ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) - -(prepend-tasks #'javac compile) + (io/copy (io/file (str srcdir "/src/google/protobuf/descriptor.proto")) + (io/file proto-files "descriptor.proto"))) + (protoc project ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) (defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." :help-arglists '([subtask & args]) @@ -147,8 +139,8 @@ ([project] (println (help-for "protobuf"))) ([project subtask & args] (case subtask - "proto" (apply google args) + "google" (apply google project args) "fetch" (apply fetch args) "install" (apply install args) "uninstall" (apply uninstall args) - "compile" (apply compile args)))) \ No newline at end of file + "compile" (apply compile-protos project args)))) \ No newline at end of file From cbb9720e8110f39d3303d689fc23662500b868f1 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 23 Nov 2011 15:21:18 -0800 Subject: [PATCH 160/525] compile works now --- src/leiningen/protobuf.clj | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index b18d74b..a831b2d 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -1,4 +1,5 @@ (ns leiningen.protobuf + (:refer-clojure :exclude [compile]) (:use [clojure.java.shell :only [sh]] [clojure.string :only [join]] [leiningen.help :only [help-for]] @@ -16,7 +17,7 @@ "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) -(defn- unzip [source target-dir] +(defn- unzip [source target-dir] (let [zip (ZipFile. source) entries (enumeration-seq (.entries zip)) target-file #(io/file target-dir (.getName %))] @@ -68,7 +69,9 @@ (flush) (join (.readPassword (System/console)))) -(defn fetch [] +(defn fetch + "Fetch protocol-buffer source and unzip it." + [] (when-not (.exists (io/file srcdir)) (with-open [stream (.openStream url)] (io/copy stream (io/file zipfile))) @@ -104,6 +107,7 @@ :dir srcdir :in password))))) (defn protoc + "Create .java and .class files from the provided .proto files." ([project protos] (protoc project protos "build/protosrc")) ([project protos dest] (when (or (> (modtime "proto") (modtime dest)) @@ -117,20 +121,26 @@ :dir "proto")) (javac (assoc project :java-source-path dest))))) -(defn compile-protos - "Compile protocol buffer files located in proto dir." - ([project] (compile-protos project (proto-files (io/file "proto")))) - ([project files] - (install) - (protoc project files))) - -(defn google +(defn compile-google-protobuf + "Compile com.google.protobuf.*" [project] (let [proto-files (io/file "proto/google/protobuf")] (.mkdirs proto-files) (io/copy (io/file (str srcdir "/src/google/protobuf/descriptor.proto")) (io/file proto-files "descriptor.proto"))) - (protoc project ["google/protobuf/descriptor.proto"] (str srcdir "/java/src/main/java"))) + (protoc project + ["google/protobuf/descriptor.proto"] + (str srcdir "/java/src/main/java"))) + +(defn compile + "Compile protocol buffer files located in proto dir." + ([project] (compile project (proto-files (io/file "proto")))) + ([project files] + (install) + (fetch) + (when (= "protobuf" (:name project)) + (compile-google-protobuf project)) + (protoc project files))) (defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." :help-arglists '([subtask & args]) @@ -139,8 +149,6 @@ ([project] (println (help-for "protobuf"))) ([project subtask & args] (case subtask - "google" (apply google project args) - "fetch" (apply fetch args) "install" (apply install args) "uninstall" (apply uninstall args) - "compile" (apply compile-protos project args)))) \ No newline at end of file + "compile" (apply compile project args)))) \ No newline at end of file From ed0fe41e1d9ef7ebdb6838d7a75f433d32e0cd2e Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 28 Nov 2011 17:25:11 -0600 Subject: [PATCH 161/525] Use target-dir for everything. --- src/leiningen/protobuf.clj | 128 ++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 57 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index a831b2d..9c0266b 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -10,12 +10,12 @@ (:import java.util.zip.ZipFile)) (def version "2.3.0") -(def srcdir (format "lib/protobuf-%s" version)) -(def zipfile (format "lib/protobuf-%s.zip" version)) -(def url (java.net.URL. - (format - "http://protobuf.googlecode.com/files/protobuf-%s.zip" - version))) +(def srcdir (str "protobuf-" version)) +(def zipfile (format "protobuf-%s.zip" version)) + +(def url + (java.net.URL. + (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) (defn- unzip [source target-dir] (let [zip (ZipFile. source) @@ -26,27 +26,29 @@ (.mkdirs (.getParentFile f)) (io/copy (.getInputStream zip entry) f)))) -(defn- proto-dependencies "look for lines starting with import in proto-file" +(defn- proto-dependencies + "look for lines starting with import in proto-file" [proto-file] (for [line (line-seq (io/reader proto-file)) :when (.startsWith line "import")] (second (re-matches #".*\"(.*)\".*" line)))) -(defn extract-dependencies "extract all files proto is dependent on" - [proto-file] +(defn extract-dependencies + "extract all files proto is dependent on" + [proto-file target] (loop [files (vec (proto-dependencies proto-file))] (when-not (empty? files) (let [proto (peek files) files (pop files)] (if (or (.exists (io/file "proto" proto)) - (.exists (io/file "build/proto" proto))) + (.exists (io/file target "proto" proto))) (recur files) (let [location (str "proto/" proto) - proto-file (io/file "build" location)] + proto-file (io/file target location)] (io/copy (io/reader (io/resource location)) proto-file) (recur (into files (proto-dependencies proto-file))))))))) (defn modtime [dir] - (let [files (rest (file-seq (io/file dir)))] + (let [files (-> dir io/file file-seq rest)] (if (empty? files) 0 (apply max (map #(.lastModified %) files))))) @@ -71,73 +73,85 @@ (defn fetch "Fetch protocol-buffer source and unzip it." - [] - (when-not (.exists (io/file srcdir)) - (with-open [stream (.openStream url)] - (io/copy stream (io/file zipfile))) - (unzip (io/file zipfile) "lib"))) + [project] + (let [target (:target-dir project)] + (when-not (.exists (io/file target srcdir)) + (let [zipped (io/file target zipfile)] + (with-open [stream (.openStream url)] + (io/copy stream (io/file zipped))) + (unzip (io/file zipped) (:target-dir project)))))) (defn uninstall "Remove protoc if it is installed." - [] + [project] (when (installed?) (let [password (read-pass)] (sh "sudo" "-S" "make" "uninstall" - :dir srcdir :in (str password "\n"))))) + :dir (io/file (:target-dir project) srcdir) + :in (str password "\n"))))) (defn install "Compile and install protoc to /usr/local." - [] + [project] (when-not (installed?) - (fetch) - (when-not (.exists (io/file srcdir "src" "protoc")) - (.setExecutable (io/file srcdir "configure") true) - (.setExecutable (io/file srcdir "install-sh") true) - (println "Configuring protoc...") - (sh "./configure" :dir srcdir) - (println "Running 'make'...") - (sh "make" :dir srcdir)) - (let [password (str (read-pass) "\n") - opts {:dir srcdir :input-string (str password "\n")}] - (println "Installing...") - (if (= :linux (get-os)) - (sh "script" "-q" "-c" "sudo -S make install" "/dev/null" - :dir srcdir :in password) - (sh "sudo" "-S" "make" "install" - :dir srcdir :in password))))) + (fetch project) + (let [source (io/file (:target-dir project) srcdir)] + (when-not (.exists (io/file source "src" "protoc")) + (.setExecutable (io/file source "configure") true) + (.setExecutable (io/file source "install-sh") true) + (println "Configuring protoc...") + (sh "./configure" :dir source) + (println "Running 'make'...") + (sh "make" :dir source)) + (let [password (str (read-pass) "\n") + opts {:dir source :input-string (str password "\n")}] + (println "Installing...") + (if (= :linux (get-os)) + (sh "script" "-q" "-c" "sudo -S make install" "/dev/null" + :dir source :in password) + (sh "sudo" "-S" "make" "install" + :dir source :in password)))))) (defn protoc "Create .java and .class files from the provided .proto files." - ([project protos] (protoc project protos "build/protosrc")) + ([project protos] + (protoc project protos (io/file (:target-dir project) "protosrc"))) ([project protos dest] - (when (or (> (modtime "proto") (modtime dest)) - (> (modtime "proto") (modtime "classes"))) - (.mkdirs (io/file dest)) - (.mkdir (io/file "build" "proto")) - (doseq [proto protos] - (println "Compiling" proto "to" dest) - (extract-dependencies (io/file "proto" proto)) - (sh "protoc" proto (str "--java_out=../" dest) "-I." "-I../build/proto" - :dir "proto")) - (javac (assoc project :java-source-path dest))))) + (let [target (:target-dir project) + dest-path (.getPath dest)] + (when (or (> (modtime "proto") (modtime dest)) + (> (modtime "proto") (modtime "classes"))) + (.mkdirs dest) + (.mkdir (io/file target "proto")) + (doseq [proto protos] + (println "Compiling" proto "to" dest-path) + (extract-dependencies (io/file "proto" proto) target) + (sh "protoc" + proto + (str "--java_out=../" dest-path) + "-I." + (str "-I../" target "/proto") + :dir "proto")) + (javac (assoc project :java-source-path dest-path)))))) (defn compile-google-protobuf "Compile com.google.protobuf.*" [project] - (let [proto-files (io/file "proto/google/protobuf")] + (let [proto-files (io/file "proto/google/protobuf") + target (:target-dir project)] (.mkdirs proto-files) - (io/copy (io/file (str srcdir "/src/google/protobuf/descriptor.proto")) - (io/file proto-files "descriptor.proto"))) - (protoc project - ["google/protobuf/descriptor.proto"] - (str srcdir "/java/src/main/java"))) + (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") + (io/file proto-files "descriptor.proto")) + (protoc project + ["google/protobuf/descriptor.proto"] + (io/file target srcdir "java/src/main/java")))) (defn compile "Compile protocol buffer files located in proto dir." ([project] (compile project (proto-files (io/file "proto")))) ([project files] - (install) - (fetch) + (install project) + (fetch project) (when (= "protobuf" (:name project)) (compile-google-protobuf project)) (protoc project files))) @@ -149,6 +163,6 @@ ([project] (println (help-for "protobuf"))) ([project subtask & args] (case subtask - "install" (apply install args) - "uninstall" (apply uninstall args) + "install" (apply install project args) + "uninstall" (apply uninstall project args) "compile" (apply compile project args)))) \ No newline at end of file From 69a018e83c8ad8bf932f90e7f3670be29aad0a88 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 28 Nov 2011 17:25:22 -0600 Subject: [PATCH 162/525] Add protobuf-* and .lein* to gitignore. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a2e3401..2777733 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.lein* .cake *~ lib @@ -8,3 +9,4 @@ pom.xml *.class proto/google/protobuf/descriptor.proto docs +protobuf-* \ No newline at end of file From 00e8b36b5e95e7b8ed6d3b6d840f8b872a15119a Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 28 Nov 2011 17:52:13 -0600 Subject: [PATCH 163/525] More logging, less ellipsis. --- src/leiningen/protobuf.clj | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 9c0266b..75ddf84 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -77,9 +77,11 @@ (let [target (:target-dir project)] (when-not (.exists (io/file target srcdir)) (let [zipped (io/file target zipfile)] + (println "Downloading" zipfile) (with-open [stream (.openStream url)] (io/copy stream (io/file zipped))) - (unzip (io/file zipped) (:target-dir project)))))) + (println "Unzipping" zipfile "to" target) + (unzip (io/file zipped) target))))) (defn uninstall "Remove protoc if it is installed." @@ -99,13 +101,13 @@ (when-not (.exists (io/file source "src" "protoc")) (.setExecutable (io/file source "configure") true) (.setExecutable (io/file source "install-sh") true) - (println "Configuring protoc...") + (println "Configuring protoc") (sh "./configure" :dir source) - (println "Running 'make'...") + (println "Running 'make'") (sh "make" :dir source)) + (println "Installing") (let [password (str (read-pass) "\n") opts {:dir source :input-string (str password "\n")}] - (println "Installing...") (if (= :linux (get-os)) (sh "script" "-q" "-c" "sudo -S make install" "/dev/null" :dir source :in password) From aade3e87dc5bdb43202788652ab95764b00a0f62 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 16:23:11 -0600 Subject: [PATCH 164/525] io version inc. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 5ab8627..2bf7c2d 100644 --- a/project.clj +++ b/project.clj @@ -1,11 +1,11 @@ -(defproject protobuf "0.5.0-beta4" +(defproject protobuf "0.5.0-beta5" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] [useful "0.7.4-alpha4"] [classlojure "0.6.3"]] :dev-dependencies [[gloss "0.2.0-rc1"] - [io "0.1.0-alpha1"]] + [io "0.1.0-alpha2"]] :eval-in-project true :java-source-path "build/protosrc" :jar-files ["proto"]) From 5e6d4fd90dfcbd17bb3a6b16bb7ae8e3ee4f6d94 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 16:27:29 -0600 Subject: [PATCH 165/525] Don't fetch and install. Install does that. --- src/leiningen/protobuf.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 75ddf84..31f5dc8 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -153,7 +153,6 @@ ([project] (compile project (proto-files (io/file "proto")))) ([project files] (install project) - (fetch project) (when (= "protobuf" (:name project)) (compile-google-protobuf project)) (protoc project files))) From 43330349a271abf7b3102d4fb86a015268a6d5f5 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 16:29:59 -0600 Subject: [PATCH 166/525] Fetch from compile if necessary and in the protobuf project (because we need to compile google protobufs). --- src/leiningen/protobuf.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 31f5dc8..275c01d 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -154,6 +154,7 @@ ([project files] (install project) (when (= "protobuf" (:name project)) + (fetch project) (compile-google-protobuf project)) (protoc project files))) From f26bf65dd62b00c910ea2140ca0e38eec5d9fd42 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 17:41:41 -0600 Subject: [PATCH 167/525] Fix some messed up paths. --- src/leiningen/protobuf.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 275c01d..cf33424 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -130,9 +130,9 @@ (extract-dependencies (io/file "proto" proto) target) (sh "protoc" proto - (str "--java_out=../" dest-path) + (str "--java_out=" dest-path) "-I." - (str "-I../" target "/proto") + (str "-I" target "/proto") :dir "proto")) (javac (assoc project :java-source-path dest-path)))))) From 5bba1890c8f88ac5be087db74a32cb8d3b3bf56f Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 17:43:13 -0600 Subject: [PATCH 168/525] Remove :java-source-path since it isn't used. --- project.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/project.clj b/project.clj index 2bf7c2d..207e75e 100644 --- a/project.clj +++ b/project.clj @@ -7,5 +7,4 @@ :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] :eval-in-project true - :java-source-path "build/protosrc" :jar-files ["proto"]) From 6bc739855f0568234ce7fbed07525f527277b71e Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 19:59:12 -0600 Subject: [PATCH 169/525] Make sure parent directories exist. --- src/leiningen/protobuf.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index cf33424..bfac2ba 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -44,6 +44,7 @@ (recur files) (let [location (str "proto/" proto) proto-file (io/file target location)] + (.mkdirs (.getParentFile proto-file)) (io/copy (io/reader (io/resource location)) proto-file) (recur (into files (proto-dependencies proto-file))))))))) From ad313eec2ddc6cf121397cc3ea19b260145d7059 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 19:59:19 -0600 Subject: [PATCH 170/525] Symlink protos so it ends up in the jar. --- resources/proto | 1 + 1 file changed, 1 insertion(+) create mode 120000 resources/proto diff --git a/resources/proto b/resources/proto new file mode 120000 index 0000000..4d1e2d8 --- /dev/null +++ b/resources/proto @@ -0,0 +1 @@ +../proto/ \ No newline at end of file From e14db1c818a71e6437bf5802d4ba9e8b42357095 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 20:09:54 -0600 Subject: [PATCH 171/525] Restructured things so that src/clj holds Clojure source and src/java holds Java source. Toto, we're not in Kan(Cake)sas anymore. --- project.clj | 2 ++ src/{ => clj}/leiningen/protobuf.clj | 0 src/{ => clj}/protobuf/codec.clj | 0 src/{ => clj}/protobuf/core.clj | 0 src/{ => java}/protobuf/core/PersistentProtocolBufferMap.java | 0 5 files changed, 2 insertions(+) rename src/{ => clj}/leiningen/protobuf.clj (100%) rename src/{ => clj}/protobuf/codec.clj (100%) rename src/{ => clj}/protobuf/core.clj (100%) rename src/{ => java}/protobuf/core/PersistentProtocolBufferMap.java (100%) diff --git a/project.clj b/project.clj index 207e75e..ada417b 100644 --- a/project.clj +++ b/project.clj @@ -7,4 +7,6 @@ :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] :eval-in-project true + :source-path "src/clj" + :java-source-path "src/java" :jar-files ["proto"]) diff --git a/src/leiningen/protobuf.clj b/src/clj/leiningen/protobuf.clj similarity index 100% rename from src/leiningen/protobuf.clj rename to src/clj/leiningen/protobuf.clj diff --git a/src/protobuf/codec.clj b/src/clj/protobuf/codec.clj similarity index 100% rename from src/protobuf/codec.clj rename to src/clj/protobuf/codec.clj diff --git a/src/protobuf/core.clj b/src/clj/protobuf/core.clj similarity index 100% rename from src/protobuf/core.clj rename to src/clj/protobuf/core.clj diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/java/protobuf/core/PersistentProtocolBufferMap.java similarity index 100% rename from src/protobuf/core/PersistentProtocolBufferMap.java rename to src/java/protobuf/core/PersistentProtocolBufferMap.java From b37cf9b85def590ff817a14b1b1ffc34366a769a Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 9 Dec 2011 20:10:16 -0600 Subject: [PATCH 172/525] :jar-files doesn't work in Leiningen and we don't need it anymore. --- project.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/project.clj b/project.clj index ada417b..846e730 100644 --- a/project.clj +++ b/project.clj @@ -8,5 +8,4 @@ [io "0.1.0-alpha2"]] :eval-in-project true :source-path "src/clj" - :java-source-path "src/java" - :jar-files ["proto"]) + :java-source-path "src/java") From 986669e0ac5b4223b4232e37c84640d0d87536a5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 12 Dec 2011 12:45:01 -0800 Subject: [PATCH 173/525] 0.6.0-beta1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 846e730..51d8297 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.5.0-beta5" +(defproject protobuf "0.6.0-beta1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] From 609894d7a5b759f3dd28e2dce24a2016806f4415 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 12 Dec 2011 13:49:33 -0800 Subject: [PATCH 174/525] fix tests --- test/protobuf/core_test.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 6b60a83..9c05399 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -207,7 +207,7 @@ :long {:type :float}, :deleted {:type :boolean}}] (is (= fields (protofields Foo))) - (is (= fields (protofields protobuf.core.Test$Foo))))) + (is (= fields (protofields protobuf.test.Core$Foo))))) (deftest test-nested-protofields (is (= {:year {:type :int}, @@ -236,7 +236,7 @@ (is (= {} (protodefault Foo :groups))) (is (= {} (protodefault Foo :item-map))) (is (= false (protodefault Foo :deleted))) - (is (= {} (protodefault protobuf.core.Test$Foo :groups)))) + (is (= {} (protodefault protobuf.test.Core$Foo :groups)))) (deftest test-use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] From af8a05ae111a785330fc1ad62a1d8a82b38e8139 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 13 Jan 2012 17:40:12 -0800 Subject: [PATCH 175/525] add better exceptions for encoding errors --- .../core/PersistentProtocolBufferMap.java | 38 +++++++++++++++---- test/protobuf/core_test.clj | 6 +++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/java/protobuf/core/PersistentProtocolBufferMap.java b/src/java/protobuf/core/PersistentProtocolBufferMap.java index 3fc0ac3..f629b60 100644 --- a/src/java/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/java/protobuf/core/PersistentProtocolBufferMap.java @@ -370,7 +370,7 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v name = name.toUpperCase().replaceAll("-","_"); Descriptors.EnumDescriptor enum_type = field.getEnumType(); Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); - if (enum_value == null) { + if (enum_value == null) { PrintWriter err = (PrintWriter) RT.ERR.deref(); err.format("invalid enum value %s for enum type %s\n", name, enum_type.getFullName()); } @@ -400,6 +400,28 @@ static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.F return null; } + protected void addRepeatedField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { + try { + builder.addRepeatedField(field, value); + } catch (IllegalArgumentException e) { + String msg = String.format("error adding %s to %s field %s", + value, field.getJavaType().toString().toLowerCase(), + field.getFullName()); + throw new IllegalArgumentException(msg, e); + } + } + + protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { + try { + builder.setField(field, value); + } catch (IllegalArgumentException e) { + String msg = String.format("error setting %s field %s to %s", + field.getJavaType().toString().toLowerCase(), + field.getFullName(), value); + throw new IllegalArgumentException(msg, e); + } + } + protected void addField(DynamicMessage.Builder builder, Object key, Object value) { if (key == null) return; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); @@ -412,7 +434,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value if (value instanceof Sequential && !set) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Object v = toProtoValue(field, s.first()); - builder.addRepeatedField(field, v); + addRepeatedField(builder,field, v); } } else { Keyword map_field_by = mapFieldBy(field); @@ -422,14 +444,14 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value IPersistentMap map = (IPersistentMap) e.getValue(); Object k = e.getKey(); Object v = toProtoValue(field, map.assoc(map_field_by, k).assoc(map_field_by.getName(), k)); - builder.addRepeatedField(field, v); + addRepeatedField(builder, field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; Object v = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, v); + addRepeatedField(builder, field, v); } } else if (set) { if (value instanceof IPersistentMap) { @@ -437,18 +459,18 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_item, e.getKey(), k_exists, e.getValue()}; Object v = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, v); + addRepeatedField(builder, field, v); } } else { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Object[] map = {k_item, s.first(), k_exists, true}; Object v = toProtoValue(field, new PersistentArrayMap(map)); - builder.addRepeatedField(field, v); + addRepeatedField(builder, field, v); } } } else { Object v = toProtoValue(field, value); - builder.addRepeatedField(field, v); + addRepeatedField(builder, field, v); } } } else { @@ -456,7 +478,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value if (v instanceof DynamicMessage) { v = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) v).build(); } - builder.setField(field, v); + setField(builder, field, v); } } diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 9c05399..c77816f 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -271,3 +271,9 @@ (.close out) (is (= [{:id 1, :label "foo"} {:id 2, :label "bar"} {:id 3, :label "baz"}] (protobuf-seq Foo in))))) + +(deftest test-encoding-errors + (is (thrown-with-msg? IllegalArgumentException #"error setting string field Foo.label to 8" + (protobuf Foo :label 8))) + (is (thrown-with-msg? IllegalArgumentException #"error adding 1 to string field Foo.tags" + (protobuf Foo :tags [1 2 3])))) \ No newline at end of file From 647f52bde81049b4a3a2e2a8187d67fa71a9313f Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 13 Jan 2012 18:06:46 -0800 Subject: [PATCH 176/525] 0.6.0-beta2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 51d8297..d01c760 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta1" +(defproject protobuf "0.6.0-beta2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] From 612cadc5b3cbcae53c9e39d2178c2afd231d08f5 Mon Sep 17 00:00:00 2001 From: Paul Ingles Date: Wed, 18 Jan 2012 16:06:10 +0000 Subject: [PATCH 177/525] allow protocol buffer source files to be specified in the project's :proto-path --- src/clj/leiningen/protobuf.clj | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/clj/leiningen/protobuf.clj b/src/clj/leiningen/protobuf.clj index bfac2ba..66abbd0 100644 --- a/src/clj/leiningen/protobuf.clj +++ b/src/clj/leiningen/protobuf.clj @@ -121,20 +121,21 @@ (protoc project protos (io/file (:target-dir project) "protosrc"))) ([project protos dest] (let [target (:target-dir project) - dest-path (.getPath dest)] - (when (or (> (modtime "proto") (modtime dest)) - (> (modtime "proto") (modtime "classes"))) + dest-path (.getPath dest) + proto-path (or (:proto-path project) "proto")] + (when (or (> (modtime proto-path) (modtime dest)) + (> (modtime proto-path) (modtime "classes"))) (.mkdirs dest) (.mkdir (io/file target "proto")) (doseq [proto protos] (println "Compiling" proto "to" dest-path) - (extract-dependencies (io/file "proto" proto) target) + (extract-dependencies (io/file proto-path proto) target) (sh "protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" target "/proto") - :dir "proto")) + :dir proto-path)) (javac (assoc project :java-source-path dest-path)))))) (defn compile-google-protobuf @@ -151,7 +152,8 @@ (defn compile "Compile protocol buffer files located in proto dir." - ([project] (compile project (proto-files (io/file "proto")))) + ([project] + (compile project (proto-files (io/file (or (:proto-path project) "proto"))))) ([project files] (install project) (when (= "protobuf" (:name project)) From 1a6d33d1ede579632e932b2036cad838a00cf279 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 13 Dec 2011 14:53:48 -0600 Subject: [PATCH 178/525] Restructure the project again since the :java-source-path in :source-path + jar Leiningen bug was fixed. --- project.clj | 7 +++++-- src/{clj => }/leiningen/protobuf.clj | 2 ++ src/{clj => }/protobuf/codec.clj | 0 src/{clj => }/protobuf/core.clj | 0 .../protobuf/core/PersistentProtocolBufferMap.java | 0 5 files changed, 7 insertions(+), 2 deletions(-) rename src/{clj => }/leiningen/protobuf.clj (99%) rename src/{clj => }/protobuf/codec.clj (100%) rename src/{clj => }/protobuf/core.clj (100%) rename src/{java => }/protobuf/core/PersistentProtocolBufferMap.java (100%) diff --git a/project.clj b/project.clj index d01c760..8f7e1d8 100644 --- a/project.clj +++ b/project.clj @@ -7,5 +7,8 @@ :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] :eval-in-project true - :source-path "src/clj" - :java-source-path "src/java") + ;; Bug in the current 1.x branch of Leiningen causes + ;; jar to implicitly clean no matter what, wiping stuff. + ;; This prevents that. + :disable-implicit-clean true + :java-source-path "src") diff --git a/src/clj/leiningen/protobuf.clj b/src/leiningen/protobuf.clj similarity index 99% rename from src/clj/leiningen/protobuf.clj rename to src/leiningen/protobuf.clj index 66abbd0..ab45424 100644 --- a/src/clj/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -138,6 +138,8 @@ :dir proto-path)) (javac (assoc project :java-source-path dest-path)))))) + + (defn compile-google-protobuf "Compile com.google.protobuf.*" [project] diff --git a/src/clj/protobuf/codec.clj b/src/protobuf/codec.clj similarity index 100% rename from src/clj/protobuf/codec.clj rename to src/protobuf/codec.clj diff --git a/src/clj/protobuf/core.clj b/src/protobuf/core.clj similarity index 100% rename from src/clj/protobuf/core.clj rename to src/protobuf/core.clj diff --git a/src/java/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java similarity index 100% rename from src/java/protobuf/core/PersistentProtocolBufferMap.java rename to src/protobuf/core/PersistentProtocolBufferMap.java From b984f6e3bf07aa8e14365f407491d218082348e1 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 13 Dec 2011 16:10:47 -0600 Subject: [PATCH 179/525] Use fs for file systemy things rather than duplicating functionality. --- project.clj | 6 +++--- src/leiningen/protobuf.clj | 20 ++++++-------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/project.clj b/project.clj index 8f7e1d8..48f814e 100644 --- a/project.clj +++ b/project.clj @@ -3,12 +3,12 @@ :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] [useful "0.7.4-alpha4"] - [classlojure "0.6.3"]] + [fs "1.0.0"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] - :eval-in-project true + :eval-in-leiningen true ;; Bug in the current 1.x branch of Leiningen causes ;; jar to implicitly clean no matter what, wiping stuff. ;; This prevents that. :disable-implicit-clean true - :java-source-path "src") + :java-source-path "src") \ No newline at end of file diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index ab45424..3e26b4b 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -6,7 +6,8 @@ [leiningen.javac :only [javac]] [leiningen.util.paths :only [get-os]] [leiningen.core :only [prepend-tasks]]) - (:require [clojure.java.io :as io]) + (:require [clojure.java.io :as io] + [fs.core :as fs]) (:import java.util.zip.ZipFile)) (def version "2.3.0") @@ -17,15 +18,6 @@ (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) -(defn- unzip [source target-dir] - (let [zip (ZipFile. source) - entries (enumeration-seq (.entries zip)) - target-file #(io/file target-dir (.getName %))] - (doseq [entry entries :when (not (.isDirectory entry)) - :let [f (target-file entry)]] - (.mkdirs (.getParentFile f)) - (io/copy (.getInputStream zip entry) f)))) - (defn- proto-dependencies "look for lines starting with import in proto-file" [proto-file] @@ -52,7 +44,7 @@ (let [files (-> dir io/file file-seq rest)] (if (empty? files) 0 - (apply max (map #(.lastModified %) files))))) + (apply max (map fs/mod-time files))))) (defn proto-file? [file] (let [name (.getName file)] @@ -82,7 +74,7 @@ (with-open [stream (.openStream url)] (io/copy stream (io/file zipped))) (println "Unzipping" zipfile "to" target) - (unzip (io/file zipped) target))))) + (fs/unzip zipped target))))) (defn uninstall "Remove protoc if it is installed." @@ -100,8 +92,8 @@ (fetch project) (let [source (io/file (:target-dir project) srcdir)] (when-not (.exists (io/file source "src" "protoc")) - (.setExecutable (io/file source "configure") true) - (.setExecutable (io/file source "install-sh") true) + (fs/chmod "+x" (io/file source "configure")) + (fs/chmod "+x" (io/file source "install-sh")) (println "Configuring protoc") (sh "./configure" :dir source) (println "Running 'make'") From 746c13a0a4d37a1b21d1bec7304839288674e8ae Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 18 Jan 2012 17:14:05 -0600 Subject: [PATCH 180/525] Hook the javac task so that things are compiled when it is ran. --- project.clj | 1 + src/leiningen/protobuf.clj | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 48f814e..facee20 100644 --- a/project.clj +++ b/project.clj @@ -6,6 +6,7 @@ [fs "1.0.0"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] + :hooks [leiningen.protobuf] :eval-in-leiningen true ;; Bug in the current 1.x branch of Leiningen causes ;; jar to implicitly clean no matter what, wiping stuff. diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 3e26b4b..df77980 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -5,7 +5,7 @@ [leiningen.help :only [help-for]] [leiningen.javac :only [javac]] [leiningen.util.paths :only [get-os]] - [leiningen.core :only [prepend-tasks]]) + [robert.hooke :only [add-hook]]) (:require [clojure.java.io :as io] [fs.core :as fs]) (:import java.util.zip.ZipFile)) @@ -14,6 +14,8 @@ (def srcdir (str "protobuf-" version)) (def zipfile (format "protobuf-%s.zip" version)) +(def ^{:dynamic true} *compile?* true) + (def url (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) @@ -128,9 +130,8 @@ "-I." (str "-I" target "/proto") :dir proto-path)) - (javac (assoc project :java-source-path dest-path)))))) - - + (binding [*compile?* false] + (javac (assoc project :java-source-path dest-path))))))) (defn compile-google-protobuf "Compile com.google.protobuf.*" @@ -155,6 +156,12 @@ (compile-google-protobuf project)) (protoc project files))) +(add-hook #'javac + (fn [f & args] + (when *compile?* + (compile (first args))) + (apply f args))) + (defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." :help-arglists '([subtask & args]) :subtasks [#'fetch #'install #'uninstall #'compile]} @@ -164,4 +171,4 @@ (case subtask "install" (apply install project args) "uninstall" (apply uninstall project args) - "compile" (apply compile project args)))) \ No newline at end of file + "compile" (apply compile project args)))) From 72fd3720a2cdd78ee31804658e1365952614ed32 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 18 Jan 2012 17:40:44 -0600 Subject: [PATCH 181/525] Inc beta. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index facee20..9bd89fe 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta2" +(defproject protobuf "0.6.0-beta3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.2"] @@ -12,4 +12,4 @@ ;; jar to implicitly clean no matter what, wiping stuff. ;; This prevents that. :disable-implicit-clean true - :java-source-path "src") \ No newline at end of file + :java-source-path "src") From e7317540c01ee69dc703769a24c1381164d72565 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 18 Jan 2012 19:20:20 -0600 Subject: [PATCH 182/525] Added a .lein-classpath to fix `lein test` This was necessary because we :eval-in-leiningen. --- .gitignore | 6 ++++-- .lein-classpath | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .lein-classpath diff --git a/.gitignore b/.gitignore index 2777733..2b9dd56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -.lein* +.lein-deps-sum +.lein-failures .cake *~ lib @@ -9,4 +10,5 @@ pom.xml *.class proto/google/protobuf/descriptor.proto docs -protobuf-* \ No newline at end of file +protobuf-* +protosrc diff --git a/.lein-classpath b/.lein-classpath new file mode 100644 index 0000000..90b07e9 --- /dev/null +++ b/.lein-classpath @@ -0,0 +1 @@ +classes From 60823e7d4a1e6d3dbe82a5fa6fa1a45ab9e72f45 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 19 Jan 2012 16:34:11 -0600 Subject: [PATCH 183/525] Rewrote a portion of the README to reflect the swich to Leiningen tasks. --- README.md | 154 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 93 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 070cbfd..c76071d 100644 --- a/README.md +++ b/README.md @@ -6,36 +6,42 @@ they are WAY faster to serialize and deserialize than standard Clojure objects. Write a `.proto` file: - message Person { - required int32 id = 1; - required string name = 2; - optional string email = 3; - repeated string likes = 4; - } +```java +message Person { + required int32 id = 1; + required string name = 2; + optional string email = 3; + repeated string likes = 4; +} +``` -If you put it in the proto directory of your project, you can compile it with cake: +If you put it in the proto directory of your project, you can compile it with [leiningen](https://github.com/technomancy/leiningen): - cake proto example.proto +``` +lein protobuf compile example.proto +``` Now you can use the protocol buffer in clojure: - (use 'protobuf) - (defprotobuf Person Example$Person) +```clojure +(use 'protobuf) +(defprotobuf Person Example$Person) - (def p (protobuf Person :id 4 :name "Bob" :email "bob@example.com")) - => {:id 4, :name "Bob", :email "bob@example.com"} +(def p (protobuf Person :id 4 :name "Bob" :email "bob@example.com")) +=> {:id 4, :name "Bob", :email "bob@example.com"} - (assoc p :name "Bill")) - => {:id 4, :name "Bill", :email "bob@example.com"} +(assoc p :name "Bill")) +=> {:id 4, :name "Bill", :email "bob@example.com"} - (assoc p :likes ["climbing" "running" "jumping"]) - => {:id 4, name "Bob", :email "bob@example.com", :likes ["climbing" "running" "jumping"]} +(assoc p :likes ["climbing" "running" "jumping"]) +=> {:id 4, name "Bob", :email "bob@example.com", :likes ["climbing" "running" "jumping"]} - (def b (protobuf-dump p)) - => # +(def b (protobuf-dump p)) +=> # - (protobuf-load Person b) - => {:id 4, :name "Bob", :email "bob@example.com"} +(protobuf-load Person b) +=> {:id 4, :name "Bob", :email "bob@example.com"} +```clojure A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. @@ -46,61 +52,87 @@ Clojure-protobuf supports extensions to protocol buffers which provide sets and repeated fields. You can also provide metadata on protobuf fields using clojure syntax. To use these, you must import the extension file and include it when compiling. For example: - import "clojure/protobuf/extensions.proto"; - message Photo { - required int32 id = 1; - required string path = 2; - repeated string labels = 3 [(set) = true]; - repeated Attr attrs = 4 [(map) = true]; - repeated Tag tags = 5 [(map_by) = "person_id"]; - - message Attr { - required string key = 1; - optional string val = 2; - } - - message Tag { - required int32 person_id = 1; - optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; - optional int32 y_coord = 3; - optional int32 width = 4; - optional int32 height = 5; - } - } +```java +import "clojure/protobuf/extensions.proto"; +message Photo { + required int32 id = 1; + required string path = 2; + repeated string labels = 3 [(set) = true]; + repeated Attr attrs = 4 [(map) = true]; + repeated Tag tags = 5 [(map_by) = "person_id"]; + + message Attr { + required string key = 1; + optional string val = 2; + } + + message Tag { + required int32 person_id = 1; + optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; + optional int32 y_coord = 3; + optional int32 width = 4; + optional int32 height = 5; + } +} +``` Compile the file: - cake proto example.proto +``` +lein protobuf compile example.proto +``` Then you can access the maps in clojure: - (use 'protobuf) - (defprotobuf Photo Example$Photo) - (defprotobuf Tag Example$Photo$Tag) +```clojure +(use 'protobuf) +(defprotobuf Photo Example$Photo) +(defprotobuf Tag Example$Photo$Tag) - (def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} - :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} - :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}})) - => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} +(def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} + :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} + :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}})) +=> {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} - (def b (protobuf-dump p)) - => # +(def b (protobuf-dump p)) +=> # - (protobuf-load Photo b) - => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} +(protobuf-load Photo b) +=> {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} - (:x-coord (protofields Tag)) - => {:max 100.0 :min -100.0} +(:x-coord (protofields Tag)) +=> {:max 100.0 :min -100.0} +``` ## Installation -As of [Cake](http://github.com/ninjudd/cake) 0.7.0, add `protobuf` and the `cake-protobuf` plugin to your project.clj: +You'll want to use this with the Leiningen build tool. You can get it by +putting it in your `:dependencies` and/or `:dev-dependencies`. If you +want to use the Leiningen plugin portion of clojure-protobuf, it has to +be in your dev-dependencies. - :dependencies [[protobuf "LATEST"]] - :cake-plugins [[cake-protobuf "LATEST"]] +```clojure +:dev-dependencies [[protobuf "x.x.x"]] +``` -You should probably replace `"LATEST"` with whatever the latest version is, but I'm going to forget -to update this every time I bump the version, so I'm not going to put an actual number here. You can -find the lastest versions here (https://clojars.org/cake-protobuf https://clojars.org/protobuf). +Replace `"x.x.x"` with the actual latest version, which you can find on +[clojars](http://clojars.org/protobuf) + +**NOTE: clojure-protobuf requires bugfixes introduced in the 1.x branch +of Leiningen. Until the 1.7.0 release, you'll have to use Leiningen off +of the latest 1.x branch in order to use the Leiningen tasks provided +with this library.** + +## History + +The build tool tasks provided with this library were originally for the +cake build tool. In 2011, the authors of that tool decided that their +time would be better spent working on a single, canonical build tool. +Leiningen was already the standard for Clojure, so that's where we are +now. As of version 0.6.0 (which has yet to see an final release, but is +usable), all of the cake-specific functionality has been rewritten for +Leiningen. Any version before that will not work with Leiningen. + +## Getting Help If you have any questions or need help, you can find us on IRC in [#flatland](irc://irc.freenode.net/#flatland). From a5344a92722baa50931ff9ab822b8aabaeb9bc87 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 19 Jan 2012 19:36:03 -0800 Subject: [PATCH 184/525] update ordered-set to fix tests --- project.clj | 2 +- src/protobuf/core/PersistentProtocolBufferMap.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 9bd89fe..a86987b 100644 --- a/project.clj +++ b/project.clj @@ -1,7 +1,7 @@ (defproject protobuf "0.6.0-beta3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] - [ordered-set "0.2.2"] + [ordered-set "0.2.3"] [useful "0.7.4-alpha4"] [fs "1.0.0"]] :dev-dependencies [[gloss "0.2.0-rc1"] diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index f629b60..00ccdf7 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -11,6 +11,7 @@ package protobuf.core; import clojure.lang.*; +import ordered_set.core.OrderedSet; import java.util.*; import java.io.InputStream; import java.io.OutputStream; @@ -299,7 +300,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); - ITransientSet set = (ITransientSet) PersistentOrderedSet.EMPTY.asTransient(); + ITransientSet set = (ITransientSet) OrderedSet.EMPTY.asTransient(); while (iterator.hasNext()) { DynamicMessage message = (DynamicMessage) iterator.next(); Object item = fromProtoValue(item_field, message.getField(item_field)); From ca2a0c6159979f6a8edefbb45c93d5ba6c001d1a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 19 Jan 2012 19:36:23 -0800 Subject: [PATCH 185/525] 0.6.0-beta4 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index a86987b..f07896e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta3" +(defproject protobuf "0.6.0-beta4" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From ab8ee723e2a8e26fe1a392c2f7e37ec9bf5eadd2 Mon Sep 17 00:00:00 2001 From: Paul Ingles Date: Wed, 18 Jan 2012 16:25:48 +0000 Subject: [PATCH 186/525] capture result of calling protoc- print error messages if the compiler fails --- src/leiningen/protobuf.clj | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index df77980..f94d014 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -122,16 +122,19 @@ (.mkdirs dest) (.mkdir (io/file target "proto")) (doseq [proto protos] - (println "Compiling" proto "to" dest-path) + (print "Compiling" proto "to" dest-path "... ") (extract-dependencies (io/file proto-path proto) target) - (sh "protoc" - proto - (str "--java_out=" dest-path) - "-I." - (str "-I" target "/proto") - :dir proto-path)) - (binding [*compile?* false] - (javac (assoc project :java-source-path dest-path))))))) + (let [protoc-result (sh "protoc" + proto + (str "--java_out=" dest-path) + "-I." + (str "-I" target "/proto") + :dir proto-path)] + (if (= (:exit protoc-result) 0) + (println "Success") + (do (println "Error") + (println (:err protoc-result)))))) + (javac (assoc project :java-source-path dest-path)))))) (defn compile-google-protobuf "Compile com.google.protobuf.*" From d803b36c4273101cc3efdbbec6962cc0375fae93 Mon Sep 17 00:00:00 2001 From: Paul Ingles Date: Wed, 18 Jan 2012 17:01:11 +0000 Subject: [PATCH 187/525] print the protoc command thats run, makes it easier to figure out what happened; fix another proto-path issue --- src/leiningen/protobuf.clj | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index f94d014..ba27c39 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -122,18 +122,13 @@ (.mkdirs dest) (.mkdir (io/file target "proto")) (doseq [proto protos] - (print "Compiling" proto "to" dest-path "... ") + (println "Compiling" proto "to" dest-path "... ") (extract-dependencies (io/file proto-path proto) target) - (let [protoc-result (sh "protoc" - proto - (str "--java_out=" dest-path) - "-I." - (str "-I" target "/proto") - :dir proto-path)] - (if (= (:exit protoc-result) 0) - (println "Success") - (do (println "Error") - (println (:err protoc-result)))))) + (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" target proto-path)]] + (println (apply str " > " (interpose " " args))) + (let [protoc-result (apply sh (concat args [:dir proto-path]))] + (if (not (= (:exit protoc-result) 0)) + (println " > ERROR: " (:err protoc-result)))))) (javac (assoc project :java-source-path dest-path)))))) (defn compile-google-protobuf From c67b20ed1888462d6f4fd0c0b10f7cac6fa33a77 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 25 Jan 2012 22:58:58 -0600 Subject: [PATCH 188/525] Switch to conch for shelling out. --- project.clj | 3 ++- src/leiningen/protobuf.clj | 44 +++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/project.clj b/project.clj index f07896e..4475eb6 100644 --- a/project.clj +++ b/project.clj @@ -3,7 +3,8 @@ :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] [useful "0.7.4-alpha4"] - [fs "1.0.0"]] + [fs "1.0.0"] + [conch "0.2.0"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] :hooks [leiningen.protobuf] diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index ba27c39..b80cdd5 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -1,13 +1,13 @@ (ns leiningen.protobuf (:refer-clojure :exclude [compile]) - (:use [clojure.java.shell :only [sh]] - [clojure.string :only [join]] + (:use [clojure.string :only [join]] [leiningen.help :only [help-for]] [leiningen.javac :only [javac]] [leiningen.util.paths :only [get-os]] [robert.hooke :only [add-hook]]) (:require [clojure.java.io :as io] - [fs.core :as fs]) + [fs.core :as fs] + [conch.core :as sh]) (:import java.util.zip.ZipFile)) (def version "2.3.0") @@ -58,7 +58,7 @@ (.substring (.getPath file) (inc (count (.getPath dir)))))) (defn installed? [] - (try (.contains (:out (sh "protoc" "--version")) version) + (try (.contains (sh/stream-to-string (sh/proc "protoc" "--version") :out) version) (catch java.io.IOException e))) (defn read-pass [] @@ -82,10 +82,11 @@ "Remove protoc if it is installed." [project] (when (installed?) - (let [password (read-pass)] - (sh "sudo" "-S" "make" "uninstall" - :dir (io/file (:target-dir project) srcdir) - :in (str password "\n"))))) + (let [password (read-pass) + proc (sh/proc "sudo" "-S" "make" "uninstall" + :dir (io/file (:target-dir project) srcdir))] + (sh/feed-from-string proc (str password "\n")) + (sh/stream-to-out proc :out)))) (defn install "Compile and install protoc to /usr/local." @@ -97,17 +98,19 @@ (fs/chmod "+x" (io/file source "configure")) (fs/chmod "+x" (io/file source "install-sh")) (println "Configuring protoc") - (sh "./configure" :dir source) + (sh/stream-to-out (sh/proc "./configure" :dir source) :out) (println "Running 'make'") - (sh "make" :dir source)) + (sh/stream-to-out (sh/proc "make" :dir source) :out)) (println "Installing") (let [password (str (read-pass) "\n") - opts {:dir source :input-string (str password "\n")}] - (if (= :linux (get-os)) - (sh "script" "-q" "-c" "sudo -S make install" "/dev/null" - :dir source :in password) - (sh "sudo" "-S" "make" "install" - :dir source :in password)))))) + opts {:dir source :input-string (str password "\n")} + proc (if (= :linux (get-os)) + (sh/proc "script" "-q" "-c" "sudo -S make install" "/dev/null" + :dir source) + (sh/proc "sudo" "-S" "make" "install" + :dir source))] + (sh/feed-from-string proc password) + (sh/stream-to-out proc :out))))) (defn protoc "Create .java and .class files from the provided .proto files." @@ -126,10 +129,11 @@ (extract-dependencies (io/file proto-path proto) target) (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" target proto-path)]] (println (apply str " > " (interpose " " args))) - (let [protoc-result (apply sh (concat args [:dir proto-path]))] - (if (not (= (:exit protoc-result) 0)) - (println " > ERROR: " (:err protoc-result)))))) - (javac (assoc project :java-source-path dest-path)))))) + (let [protoc-result (apply sh/proc (concat args [:dir proto-path]))] + (if (not (= (sh/exit-code protoc-result) 0)) + (println " > ERROR: " (sh/stream-to-string protoc-result :err)))))) + (binding [*compile?* false] + (javac (assoc project :java-source-path dest-path))))))) (defn compile-google-protobuf "Compile com.google.protobuf.*" From 4f2d8e0c2491c52456d3c31f37803fa34a69c4e6 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 26 Jan 2012 17:41:54 -0600 Subject: [PATCH 189/525] 0.6.0-beta5 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4475eb6..42d2b8c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta4" +(defproject protobuf "0.6.0-beta5" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From fa2d36e2f980f55c2a53376caeedf48c17163c4b Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 26 Jan 2012 17:42:28 -0600 Subject: [PATCH 190/525] Pass files to compile properly. Fixes #16. --- src/leiningen/protobuf.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index b80cdd5..9d0cc90 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -151,7 +151,7 @@ "Compile protocol buffer files located in proto dir." ([project] (compile project (proto-files (io/file (or (:proto-path project) "proto"))))) - ([project files] + ([project & files] (install project) (when (= "protobuf" (:name project)) (fetch project) From f7228ab87c0fa3a25f45f79c7ba1a7977bff4bd7 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 26 Jan 2012 17:42:45 -0600 Subject: [PATCH 191/525] 0.6.0-beta6 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 42d2b8c..5acf391 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta5" +(defproject protobuf "0.6.0-beta6" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From 8925a8ae5098a26d4b487634b012fd0f77b38ef0 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 26 Jan 2012 17:51:22 -0600 Subject: [PATCH 192/525] apply compile because, as it turns out, a lazy seq isn't a file. --- src/leiningen/protobuf.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 9d0cc90..3ce526b 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -150,7 +150,7 @@ (defn compile "Compile protocol buffer files located in proto dir." ([project] - (compile project (proto-files (io/file (or (:proto-path project) "proto"))))) + (apply compile project (proto-files (io/file (or (:proto-path project) "proto"))))) ([project & files] (install project) (when (= "protobuf" (:name project)) From 5b94eed6d750f8820a141789b3c1b7ca7caf31fc Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 26 Jan 2012 23:49:20 -0600 Subject: [PATCH 193/525] Added a failing test for example.proto --- proto/protobuf/test/example.proto | 25 +++++++++++++++++++++++++ test/protobuf/example_test.clj | 11 +++++++++++ 2 files changed, 36 insertions(+) create mode 100644 proto/protobuf/test/example.proto create mode 100644 test/protobuf/example_test.clj diff --git a/proto/protobuf/test/example.proto b/proto/protobuf/test/example.proto new file mode 100644 index 0000000..914b2d2 --- /dev/null +++ b/proto/protobuf/test/example.proto @@ -0,0 +1,25 @@ +import "protobuf/core/extensions.proto"; + +option java_package = "protobuf.test"; +option java_outer_classname = "Example"; + +message Photo { + required int32 id = 1; + required string path = 2; + repeated string labels = 3 [(set) = true]; + repeated Attr attrs = 4 [(map) = true]; + repeated Tag tags = 5 [(map_by) = "person_id"]; + + message Attr { + required string key = 1; + optional string val = 2; + } + + message Tag { + required int32 person_id = 1; + optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; + optional int32 y_coord = 3; + optional int32 width = 4; + optional int32 height = 5; + } +} diff --git a/test/protobuf/example_test.clj b/test/protobuf/example_test.clj new file mode 100644 index 0000000..f3886b9 --- /dev/null +++ b/test/protobuf/example_test.clj @@ -0,0 +1,11 @@ +(ns protobuf.example-test + (:use protobuf.core clojure.test)) + +(def Photo (protodef protobuf.test.Example$Photo)) +(def Tag (protodef protobuf.test.Example$Photo$Tag)) + +;; Haven't seen correct output so I have nothing to compare to. +(deftest example-test + (is (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} + :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} + :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}}))) From 0359c39ded063b82b4eb349ce9eb9d499d2a1def Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 27 Jan 2012 12:57:46 -0800 Subject: [PATCH 194/525] fix example test --- proto/protobuf/test/example.proto | 7 ++++++- test/protobuf/example_test.clj | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/proto/protobuf/test/example.proto b/proto/protobuf/test/example.proto index 914b2d2..7c568b5 100644 --- a/proto/protobuf/test/example.proto +++ b/proto/protobuf/test/example.proto @@ -6,10 +6,15 @@ option java_outer_classname = "Example"; message Photo { required int32 id = 1; required string path = 2; - repeated string labels = 3 [(set) = true]; + repeated Label labels = 3 [(set) = true]; repeated Attr attrs = 4 [(map) = true]; repeated Tag tags = 5 [(map_by) = "person_id"]; + message Label { + required string item = 1; + required bool exists = 2; + } + message Attr { required string key = 1; optional string val = 2; diff --git a/test/protobuf/example_test.clj b/test/protobuf/example_test.clj index f3886b9..59ae31b 100644 --- a/test/protobuf/example_test.clj +++ b/test/protobuf/example_test.clj @@ -2,7 +2,6 @@ (:use protobuf.core clojure.test)) (def Photo (protodef protobuf.test.Example$Photo)) -(def Tag (protodef protobuf.test.Example$Photo$Tag)) ;; Haven't seen correct output so I have nothing to compare to. (deftest example-test From e5f6da1f2697f3744b928fcfdce7c6bafc65f019 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 27 Jan 2012 16:41:22 -0600 Subject: [PATCH 195/525] Updated README.md --- README.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c76071d..1b028bc 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,10 @@ lein protobuf compile example.proto Now you can use the protocol buffer in clojure: ```clojure -(use 'protobuf) -(defprotobuf Person Example$Person) +(use 'protobuf.core) +(import Example$Person) + +(def Person (protodef Example$Person)) (def p (protobuf Person :id 4 :name "Bob" :email "bob@example.com")) => {:id 4, :name "Bob", :email "bob@example.com"} @@ -41,7 +43,7 @@ Now you can use the protocol buffer in clojure: (protobuf-load Person b) => {:id 4, :name "Bob", :email "bob@example.com"} -```clojure +``` A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. @@ -53,14 +55,20 @@ repeated fields. You can also provide metadata on protobuf fields using clojure use these, you must import the extension file and include it when compiling. For example: ```java -import "clojure/protobuf/extensions.proto"; +import "protobuf/core/extensions.proto"; + message Photo { required int32 id = 1; required string path = 2; - repeated string labels = 3 [(set) = true]; + repeated Label labels = 3 [(set) = true]; repeated Attr attrs = 4 [(map) = true]; repeated Tag tags = 5 [(map_by) = "person_id"]; + message Label { + required string item = 1; + required bool exists = 2; + } + message Attr { required string key = 1; optional string val = 2; @@ -86,8 +94,11 @@ Then you can access the maps in clojure: ```clojure (use 'protobuf) -(defprotobuf Photo Example$Photo) -(defprotobuf Tag Example$Photo$Tag) +(import Example$Photo) +(import Example$Photo$Tag) + +(def Photo (protodef Example$Photo)) +(def Tag (protodef Example$Photo$Tag)) (def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} From 56c4be49079f705591dc33ec66bd348a66b43970 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 27 Jan 2012 16:46:13 -0600 Subject: [PATCH 196/525] Update example-test with expected output. Also cleaned up the test significantly. --- test/protobuf/example_test.clj | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/protobuf/example_test.clj b/test/protobuf/example_test.clj index 59ae31b..46aa5fb 100644 --- a/test/protobuf/example_test.clj +++ b/test/protobuf/example_test.clj @@ -3,8 +3,9 @@ (def Photo (protodef protobuf.test.Example$Photo)) -;; Haven't seen correct output so I have nothing to compare to. +(def data {:id 7, :path "/photos/h2k3j4h9h23", :labels #{"hawaii" "family" "surfing"}, + :attrs {"color space" "RGB", "dimensions" "1632x1224", "alpha" "no"}, + :tags {4 {:person-id 4, :x-coord 607, :y-coord 813, :width 25, :height 27}}}) + (deftest example-test - (is (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} - :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} - :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}}))) + (is (= data (apply protobuf Photo (apply concat data))))) From 85f10bc10399abedeb7e3af7ecbf7f07df04a99d Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 8 Feb 2012 17:38:06 -0600 Subject: [PATCH 197/525] Update README since 1.7.0 was released. --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1b028bc..5765f33 100644 --- a/README.md +++ b/README.md @@ -129,10 +129,8 @@ be in your dev-dependencies. Replace `"x.x.x"` with the actual latest version, which you can find on [clojars](http://clojars.org/protobuf) -**NOTE: clojure-protobuf requires bugfixes introduced in the 1.x branch -of Leiningen. Until the 1.7.0 release, you'll have to use Leiningen off -of the latest 1.x branch in order to use the Leiningen tasks provided -with this library.** +**NOTE: clojure-protobuf requires at least version 1.7.0 of Leiningen. +It will not work in earlier versions.** ## History From 2696cc8bf76af362ee9e238ca3ac3ad2fd838cdd Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 8 Feb 2012 17:48:28 -0600 Subject: [PATCH 198/525] Clean output a bit. --- src/leiningen/protobuf.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 3ce526b..a777325 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -125,13 +125,12 @@ (.mkdirs dest) (.mkdir (io/file target "proto")) (doseq [proto protos] - (println "Compiling" proto "to" dest-path "... ") (extract-dependencies (io/file proto-path proto) target) (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" target proto-path)]] - (println (apply str " > " (interpose " " args))) + (println " > " (join " " args)) (let [protoc-result (apply sh/proc (concat args [:dir proto-path]))] (if (not (= (sh/exit-code protoc-result) 0)) - (println " > ERROR: " (sh/stream-to-string protoc-result :err)))))) + (println "ERROR: " (sh/stream-to-string protoc-result :err)))))) (binding [*compile?* false] (javac (assoc project :java-source-path dest-path))))))) From dee89feafd0365db55daf1d440d97115f45f3647 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 8 Feb 2012 17:50:21 -0600 Subject: [PATCH 199/525] 0.6.0-beta7. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5acf391..9811875 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta6" +(defproject protobuf "0.6.0-beta7" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From da816f7ad769ba029d2f5477d92ba2cfaf1d68b3 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 1 Mar 2012 16:53:12 -0600 Subject: [PATCH 200/525] Put generated files in target. Closes #22. --- src/leiningen/protobuf.clj | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index a777325..cb20f6b 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -20,6 +20,14 @@ (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) +;; In lein 1 there was a :target-dir key that defaulted to "." and wasn't really +;; used much. In lein 2, things that are generated by plugins and such are +;; supposed to go in :target-path. This will default to 'target'. We don't want +;; to wait for this, so we'll simulate it here. +(defn target [project] + (doto (io/file (or (:target-path project) "target")) + .mkdirs)) + (defn- proto-dependencies "look for lines starting with import in proto-file" [proto-file] @@ -33,6 +41,7 @@ (when-not (empty? files) (let [proto (peek files) files (pop files)] + ;; TODO Somehow check for :proto-path too? (if (or (.exists (io/file "proto" proto)) (.exists (io/file target "proto" proto))) (recur files) @@ -69,8 +78,8 @@ (defn fetch "Fetch protocol-buffer source and unzip it." [project] - (let [target (:target-dir project)] - (when-not (.exists (io/file target srcdir)) + (let [target (target project)] + (when-not (.exists (io/file target srcdir)) (let [zipped (io/file target zipfile)] (println "Downloading" zipfile) (with-open [stream (.openStream url)] @@ -84,7 +93,7 @@ (when (installed?) (let [password (read-pass) proc (sh/proc "sudo" "-S" "make" "uninstall" - :dir (io/file (:target-dir project) srcdir))] + :dir (io/file (target project) srcdir))] (sh/feed-from-string proc (str password "\n")) (sh/stream-to-out proc :out)))) @@ -93,7 +102,7 @@ [project] (when-not (installed?) (fetch project) - (let [source (io/file (:target-dir project) srcdir)] + (let [source (io/file (target project) srcdir)] (when-not (.exists (io/file source "src" "protoc")) (fs/chmod "+x" (io/file source "configure")) (fs/chmod "+x" (io/file source "install-sh")) @@ -115,18 +124,20 @@ (defn protoc "Create .java and .class files from the provided .proto files." ([project protos] - (protoc project protos (io/file (:target-dir project) "protosrc"))) + (protoc project protos (io/file (target project) "protosrc"))) ([project protos dest] - (let [target (:target-dir project) + (let [target (target project) + dest (.getAbsoluteFile dest) dest-path (.getPath dest) - proto-path (or (:proto-path project) "proto")] + proto-path (.getAbsoluteFile (io/file (or (:proto-path project) "proto")))] (when (or (> (modtime proto-path) (modtime dest)) (> (modtime proto-path) (modtime "classes"))) (.mkdirs dest) - (.mkdir (io/file target "proto")) + (.mkdirs proto-path) (doseq [proto protos] (extract-dependencies (io/file proto-path proto) target) - (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" target proto-path)]] + (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." + (str "-I" proto-path)]] (println " > " (join " " args)) (let [protoc-result (apply sh/proc (concat args [:dir proto-path]))] (if (not (= (sh/exit-code protoc-result) 0)) @@ -138,7 +149,7 @@ "Compile com.google.protobuf.*" [project] (let [proto-files (io/file "proto/google/protobuf") - target (:target-dir project)] + target (target project)] (.mkdirs proto-files) (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") (io/file proto-files "descriptor.proto")) From f7c12925d2627f3176d19f4a3e81408eb8618657 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 9 Feb 2012 17:34:02 -0800 Subject: [PATCH 201/525] Update to new schema type --- src/protobuf/codec.clj | 3 ++- src/protobuf/core.clj | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 1b15a24..31b9c97 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -36,4 +36,5 @@ val (protobuf proto val)))))) (fix repeated - #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none))))) + #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none)) + (with-meta {:schema {:type :map :fields (protofields proto)}})))) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 3242fd3..ca0977e 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -1,5 +1,6 @@ (ns protobuf.core - (:use [clojure.java.io :only [input-stream output-stream]] + (:use [useful.fn :only [fix]] + [clojure.java.io :only [input-stream output-stream]] [clojure.string :only [lower-case]]) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf GeneratedMessage CodedInputStream @@ -61,14 +62,16 @@ (for [^Descriptors$FieldDescriptor field (.getFields ^Descriptors$Descriptor type)] (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) field-name (PersistentProtocolBufferMap/intern (.getName field)) - field-type (keyword (lower-case (.name (.getJavaType field))))] - [field-name (merge (when (seq meta-string) (read-string meta-string)) - {:type field-type} - (when (.isRepeated field) - {:repeated true}) - (when (= :enum field-type) - {:values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) - (.. field getEnumType getValues)))}))]))))) + field-type (keyword (lower-case (.name (.getJavaType field)))) + descriptor (merge (when (seq meta-string) (read-string meta-string)) + {:type (fix field-type + #{:message} (constantly :map))} + (when (= :enum field-type) + {:values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) + (.. field getEnumType getValues)))}))] + [field-name (if (.isRepeated field) + {:type :list :item-type descriptor} + descriptor)]))))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." From 3aac8649bc85be81eba5830fcb87a8a128dd932f Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 10 Feb 2012 11:08:17 -0800 Subject: [PATCH 202/525] beta 8 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 9811875..57e897d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta7" +(defproject protobuf "0.6.0-beta8" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From 0b2b3877eab191d82b9540f38738a83fe4a4629b Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 14 Feb 2012 17:45:14 -0800 Subject: [PATCH 203/525] More aggressive error messages --- src/protobuf/codec.clj | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 31b9c97..b0baf30 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -13,8 +13,13 @@ (let [proto (protodef proto) min (alength (protobuf-dump proto {:_len 0})) max (alength (protobuf-dump proto {:_len Integer/MAX_VALUE}))] - (when-not (= min max) - (throw (Exception. "_len must be of type fixed32 or fixed64"))) + (letfn [(check [test msg] + (when test + (throw (Exception. (format "In %s: %s" (.getFullName proto) msg)))))] + (check (zero? min) + "_len field is required for repeated protobufs") + (check (= min max) + "_len must be of type fixed32 or fixed64")) (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) #(hash-map :_len %) :_len))) From 43be0038bfa760af0d506705551c0bbe1574131e Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 15 Feb 2012 12:11:22 -0800 Subject: [PATCH 204/525] Only define the length field once --- src/protobuf/codec.clj | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index b0baf30..c5e8ea5 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -9,20 +9,22 @@ (declare protobuf-codec) -(defn length-prefix [proto] - (let [proto (protodef proto) - min (alength (protobuf-dump proto {:_len 0})) - max (alength (protobuf-dump proto {:_len Integer/MAX_VALUE}))] - (letfn [(check [test msg] - (when test - (throw (Exception. (format "In %s: %s" (.getFullName proto) msg)))))] - (check (zero? min) - "_len field is required for repeated protobufs") - (check (= min max) - "_len must be of type fixed32 or fixed64")) - (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) - #(hash-map :_len %) - :_len))) +(let [len-key :proto_length] + (defn length-prefix [proto] + (let [proto (protodef proto) + min (alength (protobuf-dump proto {len-key 0})) + max (alength (protobuf-dump proto {len-key Integer/MAX_VALUE}))] + (letfn [(check [test msg] + (when test + (throw (Exception. (format "In %s: %s %s" + (.getFullName proto) (name len-key) msg)))))] + (check (zero? min) + "field is required for repeated protobufs") + (check (= min max) + "must be of type fixed32 or fixed64")) + (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) + #(hash-map len-key %) + len-key)))) (defn protobuf-codec [proto & {:keys [validator repeated]}] (let [proto (protodef proto)] From 1cb2435937832fbe867fbd89fe567de2271ec671 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 16 Feb 2012 15:26:20 -0800 Subject: [PATCH 205/525] Fix meta/with-meta --- src/protobuf/codec.clj | 4 ++-- src/protobuf/core/PersistentProtocolBufferMap.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index c5e8ea5..9a8a97d 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -15,10 +15,10 @@ min (alength (protobuf-dump proto {len-key 0})) max (alength (protobuf-dump proto {len-key Integer/MAX_VALUE}))] (letfn [(check [test msg] - (when test + (when-not test (throw (Exception. (format "In %s: %s %s" (.getFullName proto) (name len-key) msg)))))] - (check (zero? min) + (check (pos? min) "field is required for repeated protobufs") (check (= min max) "must be of type fixed32 or fixed64")) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 00ccdf7..9511b2f 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -27,7 +27,7 @@ import com.google.protobuf.CodedOutputStream; import com.google.protobuf.GeneratedMessage; -public class PersistentProtocolBufferMap extends APersistentMap { +public class PersistentProtocolBufferMap extends APersistentMap implements IObj { public static class Def { final Descriptors.Descriptor type; ConcurrentHashMap keyword_to_field; From 372e83956a8a3f6ecc1df5716496e6b847a2edd3 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 22 Feb 2012 18:56:19 -0800 Subject: [PATCH 206/525] protobuf nested schemas. need to fix and add tests tomorrow --- proto/protobuf/test/codec.proto | 4 +- src/protobuf/core.clj | 26 +++-------- src/protobuf/schema.clj | 83 +++++++++++++++++++++++++++++++++ test/protobuf/core_test.clj | 47 ++++++++++++------- 4 files changed, 120 insertions(+), 40 deletions(-) create mode 100644 src/protobuf/schema.clj diff --git a/proto/protobuf/test/codec.proto b/proto/protobuf/test/codec.proto index 02b1a65..616f645 100644 --- a/proto/protobuf/test/codec.proto +++ b/proto/protobuf/test/codec.proto @@ -14,8 +14,8 @@ message Foo { optional Foo nested = 7; - repeated int32 revisions = 8; - optional fixed32 _len = 9; + repeated int32 revisions = 14; + optional fixed32 proto_length = 15; } message Item { diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index ca0977e..ebce268 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -1,10 +1,10 @@ (ns protobuf.core - (:use [useful.fn :only [fix]] + (:use [protobuf.schema :only [field-schema]] + [useful.fn :only [fix]] [clojure.java.io :only [input-stream output-stream]] [clojure.string :only [lower-case]]) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) - (com.google.protobuf GeneratedMessage CodedInputStream - Descriptors$Descriptor Descriptors$FieldDescriptor) + (com.google.protobuf GeneratedMessage CodedInputStream Descriptors$Descriptor) (java.io InputStream OutputStream) (clojure.lang Reflector))) @@ -54,24 +54,10 @@ (let [type ^PersistentProtocolBufferMap$Def (protodef type)] (.defaultValue type key))) -(defn protofields - "Return a map of the protobuf fields to the clojure.protobuf.Extensions metadata for each field." +(defn protobuf-schema + "Return the schema for the given protodef." [& args] - (let [type (.getMessageType ^PersistentProtocolBufferMap$Def (apply protodef args))] - (into {} - (for [^Descriptors$FieldDescriptor field (.getFields ^Descriptors$Descriptor type)] - (let [meta-string (.. field getOptions (getExtension (Extensions/meta))) - field-name (PersistentProtocolBufferMap/intern (.getName field)) - field-type (keyword (lower-case (.name (.getJavaType field)))) - descriptor (merge (when (seq meta-string) (read-string meta-string)) - {:type (fix field-type - #{:message} (constantly :map))} - (when (= :enum field-type) - {:values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) - (.. field getEnumType getValues)))}))] - [field-name (if (.isRepeated field) - {:type :list :item-type descriptor} - descriptor)]))))) + (field-schema (apply protodef args))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/src/protobuf/schema.clj b/src/protobuf/schema.clj new file mode 100644 index 0000000..b2350fd --- /dev/null +++ b/src/protobuf/schema.clj @@ -0,0 +1,83 @@ +(ns protobuf.schema + (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) + (com.google.protobuf Descriptors$Descriptor + Descriptors$FieldDescriptor + Descriptors$FieldDescriptor$Type))) + +(defn extension [ext field] + (-> field .getOptions (.getExtension ext))) + +(defn field-type [field] + (condp instance? field + Descriptors$FieldDescriptor + (if (.isRepeated field) + (condp extension field + (Extensions/counter) :counter + (Extensions/succession) :succession + (Extensions/map) :map + (Extensions/mapBy) :map-by + (Extensions/set) :set + :list) + :basic) + Descriptors$Descriptor + :struct)) + +(defn struct-schema [struct & [parents]] + (let [struct-name (.getFullName struct)] + (into {:type :struct + :name struct-name} + (when (not-any? (partial = struct-name) parents) + {:fields (into {} + (for [field (.getFields struct)] + [(PersistentProtocolBufferMap/intern (.getName field)) + (field-schema field (conj parents struct-name))]))})))) + +(defn basic-schema [field & [parents]] + (let [java-type (keyword (lower-case (.name (.getJavaType field)))) + meta-string (extension (Extensions/meta) field)] + (into (case java-type + :message (struct-schema (.getMessageType field) parents) + :enum {:type :enum + :values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) + (.. field getEnumType getValues)))} + {:type java-type}) + (when meta-string + (read-string meta-string))))) + +(defn subfield [field field-name] + (.getFieldByName (.getMessageType field) (name field-name))) + +(defmulti field-schema (fn [field & _] (field-type field))) + +(defmethod field-schema :basic [field & [parents]] + (basic-schema field parents)) + +(defmethod field-schema :list [field & [parents]] + {:type :list + :values (basic-schema field parents)}) + +(defmethod field-schema :succession [field & [parents]] + (assoc (basic-schema field parents) + :succession true)) + +(defmethod field-schema :counter [field & [parents]] + (assoc (basic-schema field parents) + :counter true)) + +(defmethod field-schema :set [field & [parents]] + {:type :set + :values (field-schema (subfield field :item) parents)}) + +(defmethod field-schema :map [field & [parents]] + {:type :map + :keys (field-schema (subfield field :key) parents) + :values (field-schema (subfield field :val) parents)}) + +(defmethod field-schema :map-by [field & [parents]] + (let [map-by (extension (Extensions/mapBy) field)] + {:type :map-by + :keys (field-schema (subfield field map-by) parents) + :values (basic-schema field parents)})) + +(defmethod field-schema :struct [field & [parents]] + (struct-schema field parents)) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index c77816f..6dd65e0 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -188,24 +188,35 @@ (is (= ["foo" "bar" ""] (get-raw p :labels)))))))) (deftest test-protofields - (let [fields {:floats {:type :float, :repeated true}, - :doubles {:type :double, :repeated true}, - :counts {:type :message, :repeated true}, - :time {:type :message, :repeated true}, - :attr-map {:type :message, :repeated true}, - :tag-set {:type :message, :repeated true}, - :item-map {:type :message, :repeated true}, - :groups {:type :message, :repeated true}, - :responses {:type :enum, :repeated true, :values #{:yes :no :maybe :not-sure}}, - :pair-map {:type :message, :repeated true}, - :foo-by-id {:type :message, :repeated true}, - :tags {:type :string, :repeated true}, - :label {:type :string, :a 1, :b 2, :c 3}, - :id {:type :int}, - :parent {:type :message}, - :lat {:type :double}, - :long {:type :float}, - :deleted {:type :boolean}}] + (let [fields {:type :struct + :name "Foo" + :fields {:parent {:type :struct}, + :floats {:type :list, :values {:type :float}}, + :doubles {:type :list, :values {:type :double}}, + :tags {:type :list, :values {:type :string}} + :tag-set {:type :set, :values {:type :string}}, + :counts {:type :map + :keys {:type :string} + :values {:type :struct}}, + :foo-by-id {:type :map + :keys {:type :integer} + :values {:type :struct, :name "Foo"}}, + :attr-map {:type :map + :keys {:type :string} + :values {:type :string}}, + :pair-map {:type :map, + :keys {:type :string} + :item {:type :struct, :name "Pair", + :fields {}}}, + :groups {:type :list, :item {:type :map}}, + :responses {:type :list, :item {:type :enum, :values #{:no :yes :maybe :not-sure}}}, + :lat {:type :double}, + :time {:type :list, :item {:type :map}}, + :label {:type :string, :a 1, :b 2, :c 3}, + :item-map {:type :list, :item {:type :map}}, + :id {:type :int}, + :long {:type :float}, + :deleted {:type :boolean},}}] (is (= fields (protofields Foo))) (is (= fields (protofields protobuf.test.Core$Foo))))) From 9a6f38188020da99f6d0e72a47e7384d94fa1094 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 23 Feb 2012 18:24:21 -0800 Subject: [PATCH 207/525] new schema tests passing --- README.md | 4 +- proto/protobuf/test/codec.proto | 1 + proto/protobuf/test/core.proto | 2 + proto/protobuf/test/example.proto | 2 + src/protobuf/codec.clj | 2 +- src/protobuf/core.clj | 11 +- .../core/PersistentProtocolBufferMap.java | 10 -- src/protobuf/schema.clj | 33 +++-- test/protobuf/core_test.clj | 130 +++++++++--------- 9 files changed, 96 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 5765f33..a386644 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Then you can access the maps in clojure: (protobuf-load Photo b) => {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} -(:x-coord (protofields Tag)) +(:x-coord (protobuf-schema Tag)) => {:max 100.0 :min -100.0} ``` @@ -127,7 +127,7 @@ be in your dev-dependencies. ``` Replace `"x.x.x"` with the actual latest version, which you can find on -[clojars](http://clojars.org/protobuf) +[clojars](http://clojars.org/protobuf) **NOTE: clojure-protobuf requires at least version 1.7.0 of Leiningen. It will not work in earlier versions.** diff --git a/proto/protobuf/test/codec.proto b/proto/protobuf/test/codec.proto index 616f645..a367e4d 100644 --- a/proto/protobuf/test/codec.proto +++ b/proto/protobuf/test/codec.proto @@ -1,3 +1,4 @@ +package protobuf.test.codec; import "protobuf/core/extensions.proto"; option java_package = "protobuf.test"; diff --git a/proto/protobuf/test/core.proto b/proto/protobuf/test/core.proto index 8b8040a..c380827 100644 --- a/proto/protobuf/test/core.proto +++ b/proto/protobuf/test/core.proto @@ -1,3 +1,5 @@ +package protobuf.test.core; + import "protobuf/core/extensions.proto"; option java_package = "protobuf.test"; diff --git a/proto/protobuf/test/example.proto b/proto/protobuf/test/example.proto index 7c568b5..8f37027 100644 --- a/proto/protobuf/test/example.proto +++ b/proto/protobuf/test/example.proto @@ -1,3 +1,5 @@ +package protobuf.test.example; + import "protobuf/core/extensions.proto"; option java_package = "protobuf.test"; diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 9a8a97d..8fe0880 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -44,4 +44,4 @@ (protobuf proto val)))))) (fix repeated #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none)) - (with-meta {:schema {:type :map :fields (protofields proto)}})))) + (with-meta {:schema (protobuf-schema proto)})))) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index ebce268..c127cf5 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -1,8 +1,7 @@ (ns protobuf.core (:use [protobuf.schema :only [field-schema]] [useful.fn :only [fix]] - [clojure.java.io :only [input-stream output-stream]] - [clojure.string :only [lower-case]]) + [clojure.java.io :only [input-stream output-stream]]) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf GeneratedMessage CodedInputStream Descriptors$Descriptor) (java.io InputStream OutputStream) @@ -48,16 +47,10 @@ ([^PersistentProtocolBufferMap$Def type k v & kvs] (PersistentProtocolBufferMap/construct type (apply array-map k v kvs)))) -(defn protodefault - "Return the default empty protobuf of the given type." - [type key] - (let [type ^PersistentProtocolBufferMap$Def (protodef type)] - (.defaultValue type key))) - (defn protobuf-schema "Return the schema for the given protodef." [& args] - (field-schema (apply protodef args))) + (field-schema (.getMessageType (apply protodef args)))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 9511b2f..bb28a14 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -103,16 +103,6 @@ public String getFullName() { public Descriptors.Descriptor getMessageType() { return type; } - - public Object defaultValue(Keyword key) { - Descriptors.FieldDescriptor field = fieldDescriptor(key); - if (field.getType() == Descriptors.FieldDescriptor.Type.MESSAGE) { - if (!field.isRepeated()) return null; - return PersistentProtocolBufferMap.fromProtoValue(field, new ArrayList()); - } else { - return PersistentProtocolBufferMap.fromProtoValue(field, field.getDefaultValue()); - } - } } final Def def; diff --git a/src/protobuf/schema.clj b/src/protobuf/schema.clj index b2350fd..e9df2da 100644 --- a/src/protobuf/schema.clj +++ b/src/protobuf/schema.clj @@ -1,11 +1,14 @@ (ns protobuf.schema + (:use [useful.fn :only [fix]] + [clojure.string :only [lower-case]]) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor Descriptors$FieldDescriptor$Type))) - (defn extension [ext field] - (-> field .getOptions (.getExtension ext))) + (-> (.getOptions field) + (.getExtension ext) + (fix string? not-empty))) (defn field-type [field] (condp instance? field @@ -22,6 +25,8 @@ Descriptors$Descriptor :struct)) +(defmulti field-schema (fn [field & _] (field-type field))) + (defn struct-schema [struct & [parents]] (let [struct-name (.getFullName struct)] (into {:type :struct @@ -35,19 +40,19 @@ (defn basic-schema [field & [parents]] (let [java-type (keyword (lower-case (.name (.getJavaType field)))) meta-string (extension (Extensions/meta) field)] - (into (case java-type - :message (struct-schema (.getMessageType field) parents) - :enum {:type :enum - :values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) - (.. field getEnumType getValues)))} - {:type java-type}) - (when meta-string - (read-string meta-string))))) + (merge (case java-type + :message (struct-schema (.getMessageType field) parents) + :enum {:type :enum + :values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) + (.. field getEnumType getValues)))} + {:type java-type}) + (when (.hasDefaultValue field) + {:default (.getDefaultValue field)}) + (when meta-string + (read-string meta-string))))) (defn subfield [field field-name] - (.getFieldByName (.getMessageType field) (name field-name))) - -(defmulti field-schema (fn [field & _] (field-type field))) + (.findFieldByName (.getMessageType field) (name field-name))) (defmethod field-schema :basic [field & [parents]] (basic-schema field parents)) @@ -75,7 +80,7 @@ (defmethod field-schema :map-by [field & [parents]] (let [map-by (extension (Extensions/mapBy) field)] - {:type :map-by + {:type :map :keys (field-schema (subfield field map-by) parents) :values (basic-schema field parents)})) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 6dd65e0..71a5d8d 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -187,67 +187,71 @@ (is (= ["foo" "bar" nil] (get p :labels))) (is (= ["foo" "bar" ""] (get-raw p :labels)))))))) -(deftest test-protofields - (let [fields {:type :struct - :name "Foo" - :fields {:parent {:type :struct}, - :floats {:type :list, :values {:type :float}}, - :doubles {:type :list, :values {:type :double}}, - :tags {:type :list, :values {:type :string}} - :tag-set {:type :set, :values {:type :string}}, - :counts {:type :map - :keys {:type :string} - :values {:type :struct}}, - :foo-by-id {:type :map - :keys {:type :integer} - :values {:type :struct, :name "Foo"}}, - :attr-map {:type :map - :keys {:type :string} - :values {:type :string}}, - :pair-map {:type :map, - :keys {:type :string} - :item {:type :struct, :name "Pair", - :fields {}}}, - :groups {:type :list, :item {:type :map}}, - :responses {:type :list, :item {:type :enum, :values #{:no :yes :maybe :not-sure}}}, - :lat {:type :double}, - :time {:type :list, :item {:type :map}}, - :label {:type :string, :a 1, :b 2, :c 3}, - :item-map {:type :list, :item {:type :map}}, - :id {:type :int}, - :long {:type :float}, - :deleted {:type :boolean},}}] - (is (= fields (protofields Foo))) - (is (= fields (protofields protobuf.test.Core$Foo))))) +(deftest test-protobuf-schema + (let [fields + {:type :struct + :name "protobuf.test.core.Foo" + :fields {:id {:default 43, :type :int} + :deleted {:default false, :type :boolean} + :lat {:type :double} + :long {:type :float} + :parent {:type :struct, :name "protobuf.test.core.Foo"} + :floats {:type :list, :values {:type :float}} + :doubles {:type :list, :values {:type :double}} + :label {:type :string, :c 3, :b 2, :a 1} + :tags {:type :list, :values {:type :string}} + :tag-set {:type :set, :values {:type :string}} + :counts {:type :map + :keys {:type :string} + :values {:type :struct, :name "protobuf.test.core.Count" + :fields {:key {:type :string} + :i {:counter true, :type :int} + :d {:counter true, :type :double}}}} + :foo-by-id {:type :map + :keys {:default 43, :type :int} + :values {:type :struct, :name "protobuf.test.core.Foo"}} + :attr-map {:type :map + :keys {:type :string} + :values {:type :string}} + :pair-map {:type :map + :keys {:type :string} + :values {:type :struct, :name "protobuf.test.core.Pair" + :fields {:key {:type :string} + :val {:type :string}}}} + :groups {:type :map + :keys {:type :string} + :values {:type :list + :values {:type :struct, :name "protobuf.test.core.Foo"}}} + :responses {:type :list + :values {:type :enum, :values #{:no :yes :maybe :not-sure}}} + :time {:type :struct, :name "protobuf.test.core.Time", :succession true + :fields {:year {:type :int} + :month {:type :int} + :day {:type :int} + :hour {:type :int} + :minute {:type :int}}} + :item-map {:type :map + :keys {:type :string} + :values {:type :struct, :name "protobuf.test.core.Item" + :fields {:item {:type :string}, + :exists {:default true, :type :boolean}}}}}}] + (is (= fields (protobuf-schema Foo))) + (is (= fields (protobuf-schema protobuf.test.Core$Foo))))) -(deftest test-nested-protofields - (is (= {:year {:type :int}, - :month {:type :int}, - :day {:type :int}, - :hour {:type :int}, - :minute {:type :int}} - (protofields Foo :time))) - (is (= {:year {:type :int}, - :month {:type :int}, - :day {:type :int}, - :hour {:type :int}, - :minute {:type :int}} - (protofields Foo :parent :foo-by-id :time)))) - -(deftest test-protodefault - (is (= 43 (protodefault Foo :id))) - (is (= 0.0 (protodefault Foo :lat))) - (is (= 0.0 (protodefault Foo :long))) - (is (= "" (protodefault Foo :label))) - (is (= [] (protodefault Foo :tags))) - (is (= nil (protodefault Foo :parent))) - (is (= [] (protodefault Foo :responses))) - (is (= #{} (protodefault Foo :tag-set))) - (is (= {} (protodefault Foo :foo-by-id))) - (is (= {} (protodefault Foo :groups))) - (is (= {} (protodefault Foo :item-map))) - (is (= false (protodefault Foo :deleted))) - (is (= {} (protodefault protobuf.test.Core$Foo :groups)))) +(comment deftest test-default-protobuf + (is (= 43 (default-protobuf Foo :id))) + (is (= 0.0 (default-protobuf Foo :lat))) + (is (= 0.0 (default-protobuf Foo :long))) + (is (= "" (default-protobuf Foo :label))) + (is (= [] (default-protobuf Foo :tags))) + (is (= nil (default-protobuf Foo :parent))) + (is (= [] (default-protobuf Foo :responses))) + (is (= #{} (default-protobuf Foo :tag-set))) + (is (= {} (default-protobuf Foo :foo-by-id))) + (is (= {} (default-protobuf Foo :groups))) + (is (= {} (default-protobuf Foo :item-map))) + (is (= false (default-protobuf Foo :deleted))) + (is (= {} (default-protobuf protobuf.test.Core$Foo :groups)))) (deftest test-use-underscores (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] @@ -260,7 +264,7 @@ (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id :pair_map :groups :doubles :floats :item_map :counts :time :lat :long} - (set (keys (protofields Foo))))) + (-> (protobuf-schema Foo) :fields keys set))) (protobuf.core.PersistentProtocolBufferMap/setUseUnderscores false))) @@ -284,7 +288,7 @@ (protobuf-seq Foo in))))) (deftest test-encoding-errors - (is (thrown-with-msg? IllegalArgumentException #"error setting string field Foo.label to 8" + (is (thrown-with-msg? IllegalArgumentException #"error setting string field protobuf.test.core.Foo.label to 8" (protobuf Foo :label 8))) - (is (thrown-with-msg? IllegalArgumentException #"error adding 1 to string field Foo.tags" + (is (thrown-with-msg? IllegalArgumentException #"error adding 1 to string field protobuf.test.core.Foo.tags" (protobuf Foo :tags [1 2 3])))) \ No newline at end of file From 68709bbef084b757f692490c2a41a95503d1dc90 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 22 Feb 2012 19:12:34 -0800 Subject: [PATCH 208/525] Fix handling of reset/length fields --- src/protobuf/codec.clj | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 8fe0880..0d7a4b6 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -9,22 +9,24 @@ (declare protobuf-codec) -(let [len-key :proto_length] - (defn length-prefix [proto] - (let [proto (protodef proto) - min (alength (protobuf-dump proto {len-key 0})) - max (alength (protobuf-dump proto {len-key Integer/MAX_VALUE}))] - (letfn [(check [test msg] - (when-not test - (throw (Exception. (format "In %s: %s %s" - (.getFullName proto) (name len-key) msg)))))] - (check (pos? min) - "field is required for repeated protobufs") - (check (= min max) - "must be of type fixed32 or fixed64")) - (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) - #(hash-map len-key %) - len-key)))) +(def ^{:private true} len-key :proto_length) +(def ^{:private true} reset-key :codec_reset) + +(defn length-prefix [proto] + (let [proto (protodef proto) + min (alength (protobuf-dump proto {len-key 0})) + max (alength (protobuf-dump proto {len-key Integer/MAX_VALUE}))] + (letfn [(check [test msg] + (when-not test + (throw (Exception. (format "In %s: %s %s" + (.getFullName proto) (name len-key) msg)))))] + (check (pos? min) + "field is required for repeated protobufs") + (check (= min max) + "must be of type fixed32 or fixed64")) + (gloss/compile-frame (gloss/finite-frame max (protobuf-codec proto)) + #(hash-map len-key %) + len-key))) (defn protobuf-codec [proto & {:keys [validator repeated]}] (let [proto (protodef proto)] @@ -43,5 +45,13 @@ val (protobuf proto val)))))) (fix repeated - #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none)) + #(gloss/compile-frame (gloss/repeated (gloss/finite-frame (length-prefix proto) %) + :prefix :none) + identity + (partial dissoc len-key))) + (gloss/compile-frame identity + (fn [data] + (-> data + (dissoc reset-key :revisions) + (with-meta (select-keys data [:revisions]))))) (with-meta {:schema (protobuf-schema proto)})))) From 7442c3f7e35a78c887770d48e70f405082e15a1a Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 27 Feb 2012 11:58:13 -0800 Subject: [PATCH 209/525] Fix metadata support --- src/protobuf/core/PersistentProtocolBufferMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index bb28a14..22597f3 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -475,7 +475,7 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value public PersistentProtocolBufferMap withMeta(IPersistentMap meta) { if (meta == meta()) return this; - return new PersistentProtocolBufferMap(meta(), def, message); + return new PersistentProtocolBufferMap(meta, def, message); } public IPersistentMap meta(){ From d2267c97f504dc2ae217ba083e1d3f57d28e9731 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 27 Feb 2012 11:58:39 -0800 Subject: [PATCH 210/525] Make .contains consistent with .getValAt --- src/protobuf/core/PersistentProtocolBufferMap.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 22597f3..d8518e8 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -489,7 +489,7 @@ public boolean containsKey(Object key) { } else if (field.isRepeated()) { return message().getRepeatedFieldCount(field) > 0; } else { - return message().hasField(field); + return message().hasField(field) || field.hasDefaultValue(); } } @@ -509,10 +509,11 @@ public Object valAt(Object key, Object notFound) { public Object getValAt(Object key, boolean use_extensions) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return null; - if (field.isRepeated() && message().getRepeatedFieldCount(field) == 0) return null; - if (!field.isRepeated() && !field.hasDefaultValue() && !message().hasField(field)) return null; - return fromProtoValue(field, message().getField(field), use_extensions); + if (containsKey(key)) { + return fromProtoValue(field, message().getField(field), use_extensions); + } else { + return null; + } } public IPersistentMap assoc(Object key, Object value) { From e44110a93ddeb01c4287b50c1803da483266a0d6 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 27 Feb 2012 13:03:23 -0800 Subject: [PATCH 211/525] Add tests for equality and meta testing --- test/protobuf/core_test.clj | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 71a5d8d..a876c65 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -60,6 +60,21 @@ (is (= nil (:tags p))) (is (= nil (:label p)))))) +(deftest test-equality + (let [m {:id 5 :tags ["fast" "shiny"] :label "nice"} + p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice") + q (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] + (is (= m p)) + (is (= q p)))) + +(deftest test-meta + (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice") + m {:foo :bar} + q (with-meta p m)] + (is (empty? (meta p))) + (is (= p q)) + (is (= m (meta q))))) + (deftest test-string-keys (let [p (protobuf Foo "id" 5 "label" "rad")] (is (= 5 (p :id))) From c23723a75f17c831e298c7092216fd659297242e Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 27 Feb 2012 13:23:33 -0800 Subject: [PATCH 212/525] Manage meta keys better --- src/protobuf/codec.clj | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 0d7a4b6..f1d3a73 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -3,6 +3,7 @@ [gloss.core.protocols :only [Reader Writer]] [gloss.core.formats :only [to-buf-seq]] [useful.fn :only [fix]] + [useful.experimental :only [lift-meta]] [clojure.java.io :only [input-stream]]) (:require io.core [gloss.core :as gloss])) @@ -44,14 +45,9 @@ (if (protobuf? val) val (protobuf proto val)))))) - (fix repeated - #(gloss/compile-frame (gloss/repeated (gloss/finite-frame (length-prefix proto) %) - :prefix :none) - identity - (partial dissoc len-key))) (gloss/compile-frame identity - (fn [data] - (-> data - (dissoc reset-key :revisions) - (with-meta (select-keys data [:revisions]))))) + #(dissoc % reset-key)) + (fix repeated + #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) + :prefix :none)) (with-meta {:schema (protobuf-schema proto)})))) From cf8c034b576545d31f148532de6822fb03ea354d Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 27 Feb 2012 14:37:39 -0800 Subject: [PATCH 213/525] New version of useful --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 57e897d..68c3f20 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] - [useful "0.7.4-alpha4"] + [useful "0.8.0-alpha1"] [fs "1.0.0"] [conch "0.2.0"]] :dev-dependencies [[gloss "0.2.0-rc1"] From d0d199425e0b23876873d782193acd5a207e3f94 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 28 Feb 2012 14:13:40 -0800 Subject: [PATCH 214/525] Fix catbytes (a test helper function). It was written with dependencies on the system's default character encoding, and wound up working only on OSX. Only tests were affected; the actual functionality worked all along. --- test/protobuf/core_test.clj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index a876c65..d77d386 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -8,7 +8,14 @@ (def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) (defn catbytes [& args] - (.getBytes (apply str (map (fn [#^bytes b] (String. b)) args)))) + (let [out-buf (byte-array (reduce + (map count args)))] + (loop [offset 0, args args] + (if-let [[^bytes array & more] (seq args)] + (let [size (count array)] + (System/arraycopy array 0 + out-buf offset size) + (recur (+ size offset) more)) + out-buf)))) (deftest test-conj (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] From a05634bb82dacc9afe95ec7f0f966fc9abb5b1bf Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 17:09:00 -0800 Subject: [PATCH 215/525] Add an extmap to protobuf maps --- .../core/PersistentProtocolBufferMap.java | 103 ++++++++++++------ 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index d8518e8..79ea7cf 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -105,11 +105,12 @@ public Descriptors.Descriptor getMessageType() { } } - final Def def; - final DynamicMessage message; - final IPersistentMap _meta; + private final Def def; + private final DynamicMessage message; + private final IPersistentMap _meta; + private final IPersistentMap ext; - DynamicMessage built_message; + private DynamicMessage built_message; static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws InvalidProtocolBufferException { DynamicMessage message = def.parseFrom(bytes); @@ -137,18 +138,35 @@ static public PersistentProtocolBufferMap construct(Def def, Object keyvals) { protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { this._meta = meta; + this.ext = null; this.def = def; this.message = null; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message) { this._meta = meta; + this.ext = null; + this.def = def; + this.message = message; + } + + protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, DynamicMessage message) { + this._meta = meta; + this.ext = ext; this.def = def; this.message = message; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage.Builder builder) { this._meta = meta; + this.ext = null; + this.def = def; + this.message = builder.build(); + } + + protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, DynamicMessage.Builder builder) { + this._meta = meta; + this.ext = ext; this.def = def; this.message = builder.build(); } @@ -413,11 +431,11 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip } } - protected void addField(DynamicMessage.Builder builder, Object key, Object value) { - if (key == null) return; + protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object key, Object value) { + if (key == null) return builder; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return; - if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) return; + if (field == null) return builder; + if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) return builder; boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { @@ -471,11 +489,13 @@ protected void addField(DynamicMessage.Builder builder, Object key, Object value } setField(builder, field, v); } + + return builder; } public PersistentProtocolBufferMap withMeta(IPersistentMap meta) { if (meta == meta()) return this; - return new PersistentProtocolBufferMap(meta, def, message); + return new PersistentProtocolBufferMap(meta, ext, def, message); } public IPersistentMap meta(){ @@ -483,6 +503,10 @@ public IPersistentMap meta(){ } public boolean containsKey(Object key) { + return protoContainsKey(key) || RT.booleanCast(RT.contains(ext, key)); + } + + private boolean protoContainsKey(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) { return false; @@ -493,9 +517,11 @@ public boolean containsKey(Object key) { } } + private static final Object sentinel = new Object(); + public IMapEntry entryAt(Object key) { - Object value = valAt(key); - return (value == null) ? null : new MapEntry(key, value); + Object value = valAt(key, sentinel); + return (value == sentinel) ? null : new MapEntry(key, value); } public Object valAt(Object key) { @@ -503,16 +529,20 @@ public Object valAt(Object key) { } public Object valAt(Object key, Object notFound) { - Object value = valAt(key); - return (value == null) ? notFound : value; + return getValAt(key, notFound, true); } public Object getValAt(Object key, boolean use_extensions) { + Object val = getValAt(key, sentinel, use_extensions); + return (val == sentinel) ? null : val; + } + + public Object getValAt(Object key, Object notFound, boolean use_extensions) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (containsKey(key)) { + if (protoContainsKey(key)) { return fromProtoValue(field, message().getField(field), use_extensions); } else { - return null; + return RT.get(ext, key, notFound); } } @@ -520,8 +550,13 @@ public IPersistentMap assoc(Object key, Object value) { DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - addField(builder, field, value); - return new PersistentProtocolBufferMap(meta(), def, builder); + if (field != null) { + addField(builder, field, value); + return new PersistentProtocolBufferMap(meta(), ext, def, builder); + } else { + return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), + def, builder); + } } public IPersistentMap assocEx(Object key, Object value) throws Exception { @@ -554,17 +589,21 @@ public PersistentProtocolBufferMap append(IPersistentMap map) { } else { proto = construct(def, map); } - return new PersistentProtocolBufferMap(meta(), def, builder().mergeFrom(proto.message())); + return new PersistentProtocolBufferMap(meta(), ext, def, builder().mergeFrom(proto.message())); } public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return this; + if (field == null) { + IPersistentMap newExt = (IPersistentMap)RT.dissoc(ext, key); + if (newExt == ext) { + return this; + } + return new PersistentProtocolBufferMap(meta(), newExt, def, builder()); + } if (field.isRequired()) throw new Exception("Can't remove required field"); - DynamicMessage.Builder builder = builder(); - builder.clearField(field); - return new PersistentProtocolBufferMap(meta(), def, builder); + return new PersistentProtocolBufferMap(meta(), ext, def, builder().clearField(field)); } public Iterator iterator() { @@ -580,24 +619,22 @@ public ISeq seq() { } public IPersistentCollection empty() { - DynamicMessage.Builder builder = builder(); - builder.clear(); - return new PersistentProtocolBufferMap(meta(), def, builder); + return new PersistentProtocolBufferMap(meta(), null, def, builder().clear()); } - static class Seq extends ASeq { - final PersistentProtocolBufferMap proto; - final MapEntry first; - final ISeq fields; + private static class Seq extends ASeq { + private final PersistentProtocolBufferMap proto; + private final MapEntry first; + private final ISeq fields; - static public Seq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ + public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ for (ISeq s = fields; s != null; s = s.next()) { Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); Keyword k = intern(field.getName()); - Object v = proto.valAt(k); - if (v != null) return new Seq(meta, proto, new MapEntry(k, v), s); + Object v = proto.valAt(k, sentinel); + if (v != sentinel) return new Seq(meta, proto, new MapEntry(k, v), s); } - return null; + return RT.seq(proto.ext); } protected Seq(IPersistentMap meta, PersistentProtocolBufferMap proto, MapEntry first, ISeq fields){ From 4172eae963a5d0359a45ee74ceb1c6a32eb93735 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 17:10:02 -0800 Subject: [PATCH 216/525] Update test for changes to protobuf. If you assoc nil into a protobuf with a nullable field, the key should be present but with a value of nil. --- test/protobuf/core_test.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index d77d386..e3afc80 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -182,20 +182,21 @@ (get-raw p :time)))))) (deftest test-nullable - (let [p (protobuf Bar :int 1 :long 330000000000 :flt 1.23 :dbl 9.87654321 :str "foo")] + (let [p (protobuf Bar :int 1 :long 330000000000 :flt 1.23 :dbl 9.87654321 :str "foo") + keyset #{:int :long :flt :dbl :str}] (is (= 1 (get p :int))) (is (= 330000000000 (get p :long))) (is (= (float 1.23) (get p :flt))) (is (= 9.87654321 (get p :dbl))) (is (= "foo" (get p :str))) - (is (= [:int :long :flt :dbl :str] (keys p))) + (is (= keyset (set (keys p)))) (let [p (append p {:int nil :long nil :flt nil :dbl nil :str nil})] (is (= nil (get p :int))) (is (= nil (get p :long))) (is (= nil (get p :flt))) (is (= nil (get p :dbl))) (is (= nil (get p :str))) - (is (= nil (keys p)))) + (is (= keyset (set (keys p))))) (testing "nullable successions" (let [p (protobuf Bar :label "foo")] (is (= "foo" (get p :label))) From 852fd2e95455fc061732bad212a1996097244033 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 17:12:41 -0800 Subject: [PATCH 217/525] Use result of addField (like a transient) instead of bashing builder in-place --- src/protobuf/core/PersistentProtocolBufferMap.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 79ea7cf..96b1d8c 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -547,15 +547,14 @@ public Object getValAt(Object key, Object notFound, boolean use_extensions) { } public IPersistentMap assoc(Object key, Object value) { - DynamicMessage.Builder builder = builder(); Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field != null) { - addField(builder, field, value); - return new PersistentProtocolBufferMap(meta(), ext, def, builder); + return new PersistentProtocolBufferMap(meta(), ext, def, + addField(builder(), field, value)); } else { return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), - def, builder); + def, builder()); } } From 85147572d635ed9bc49683c213924faaf297d8b4 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 19:00:04 -0800 Subject: [PATCH 218/525] Make sure count() includes unset fields with defaults. These will appear in a seq of the map, so they should be included in the count (and otherwise equality is broken and asymmetrical). --- src/protobuf/core/PersistentProtocolBufferMap.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 96b1d8c..43765a5 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -610,7 +610,14 @@ public Iterator iterator() { } public int count() { - return message().getAllFields().size(); + DynamicMessage message = message(); + int count = RT.count(ext); + for (Descriptors.FieldDescriptor field : def.type.getFields()) { + if (protoContainsKey(field)) { + count++; + } + } + return count; } public ISeq seq() { From 3d91fe574f875fa57aa84853fb99ee8c9cde9d52 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 19:00:35 -0800 Subject: [PATCH 219/525] Equality should be symmetrical --- test/protobuf/core_test.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index e3afc80..85a186d 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -72,6 +72,7 @@ p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice") q (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] (is (= m p)) + (is (= p m)) (is (= q p)))) (deftest test-meta From 85ad1a82bf7301b15170a9f94b2dd3985c6f8fd0 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 19:01:01 -0800 Subject: [PATCH 220/525] Comparing for equality should include the default values --- test/protobuf/core_test.clj | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 85a186d..b1f5aa0 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -59,7 +59,9 @@ (let [p (assoc p :tags "aspirin")] (is (= ["aspirin"] (:tags p)))) (let [p (assoc p :foo-by-id {3 {:label "three"} 2 {:label "two"}})] - (is (= {3 {:id 3, :label "three"} 2 {:id 2, :label "two"}} (:foo-by-id p)))))) + (is (= {3 {:id 3, :label "three", :deleted false} + 2 {:id 2, :label "two", :deleted false}} + (:foo-by-id p)))))) (deftest test-dissoc (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] @@ -68,7 +70,7 @@ (is (= nil (:label p)))))) (deftest test-equality - (let [m {:id 5 :tags ["fast" "shiny"] :label "nice"} + (let [m {:id 5 :tags ["fast" "shiny"] :label "nice" :deleted false} p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice") q (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice")] (is (= m p)) @@ -308,7 +310,9 @@ baz (protobuf Foo :id 3 :label "baz")] (protobuf-write out foo bar baz) (.close out) - (is (= [{:id 1, :label "foo"} {:id 2, :label "bar"} {:id 3, :label "baz"}] + (is (= [{:id 1, :label "foo", :deleted false} + {:id 2, :label "bar", :deleted false} + {:id 3, :label "baz", :deleted false}] (protobuf-seq Foo in))))) (deftest test-encoding-errors From 4713fa513c15995552faabcf65c8a523468e7274 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 1 Mar 2012 19:01:09 -0800 Subject: [PATCH 221/525] Add test for extmap --- test/protobuf/core_test.clj | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index b1f5aa0..c94146f 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -85,6 +85,17 @@ (is (= p q)) (is (= m (meta q))))) +(deftest test-extmap + (let [p (protobuf Foo :id 5 :tags ["fast" "shiny"] :label "nice") + m {:id 5 :tags ["fast" "shiny"] :label "nice" :deleted false} + p2 (assoc p :some-key 10) + m2 (assoc m :some-key 10)] + (is (= p m)) + (is (= p2 m2)) + (is (= m2 p2)) + (is (= (into {} m2) (into {} p2))) + (is (= (set (keys m2)) (set (keys p2)))))) + (deftest test-string-keys (let [p (protobuf Foo "id" 5 "label" "rad")] (is (= 5 (p :id))) From f0be8fa5614ebaf9840d183b5ce1b2df20ccc062 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 21 Mar 2012 16:21:27 -0700 Subject: [PATCH 222/525] pass proto-path into extract-dependencies --- src/leiningen/protobuf.clj | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index cb20f6b..39fded4 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -36,20 +36,20 @@ (defn extract-dependencies "extract all files proto is dependent on" - [proto-file target] - (loop [files (vec (proto-dependencies proto-file))] - (when-not (empty? files) - (let [proto (peek files) - files (pop files)] - ;; TODO Somehow check for :proto-path too? - (if (or (.exists (io/file "proto" proto)) - (.exists (io/file target "proto" proto))) - (recur files) - (let [location (str "proto/" proto) - proto-file (io/file target location)] - (.mkdirs (.getParentFile proto-file)) - (io/copy (io/reader (io/resource location)) proto-file) - (recur (into files (proto-dependencies proto-file))))))))) + [proto-path proto-file target] + (let [proto-file (io/file proto-path proto-file)] + (loop [files (vec (proto-dependencies proto-file))] + (when-not (empty? files) + (let [proto (peek files) + files (pop files)] + (if (or (.exists (io/file proto-path proto)) + (.exists (io/file target "proto" proto))) + (recur files) + (let [location (str "proto/" proto) + proto-file (io/file target location)] + (.mkdirs (.getParentFile proto-file)) + (io/copy (io/reader (io/resource location)) proto-file) + (recur (into files (proto-dependencies proto-file)))))))))) (defn modtime [dir] (let [files (-> dir io/file file-seq rest)] @@ -135,7 +135,7 @@ (.mkdirs dest) (.mkdirs proto-path) (doseq [proto protos] - (extract-dependencies (io/file proto-path proto) target) + (extract-dependencies proto-path proto target) (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." (str "-I" proto-path)]] (println " > " (join " " args)) From c9faea0aaffb5d5a7cc8758275c73ed8fa1127c6 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 21 Mar 2012 16:21:44 -0700 Subject: [PATCH 223/525] remove fetch subtask --- src/leiningen/protobuf.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 39fded4..84f1601 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -176,7 +176,7 @@ (defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." :help-arglists '([subtask & args]) - :subtasks [#'fetch #'install #'uninstall #'compile]} + :subtasks [#'install #'uninstall #'compile]} protobuf ([project] (println (help-for "protobuf"))) ([project subtask & args] From c8cbc6f6443bdda7b2f9af31204ff12423ff99eb Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 21 Mar 2012 18:58:25 -0500 Subject: [PATCH 224/525] Add target/proto to the include list. --- src/leiningen/protobuf.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 84f1601..034dce3 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -137,6 +137,7 @@ (doseq [proto protos] (extract-dependencies proto-path proto target) (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." + (str "-I" (.getAbsoluteFile (io/file target "proto"))) (str "-I" proto-path)]] (println " > " (join " " args)) (let [protoc-result (apply sh/proc (concat args [:dir proto-path]))] From 7c96e7aea5fd252cec31a8e22295cc3436b85701 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 21 Mar 2012 19:04:55 -0500 Subject: [PATCH 225/525] Update to beta9. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 68c3f20..1134a9f 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta8" +(defproject protobuf "0.6.0-beta9" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From d32ee176772869dd2a25185b48312103c472db38 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 21 Mar 2012 19:21:53 -0500 Subject: [PATCH 226/525] Beta10 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1134a9f..ec0a26e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta9" +(defproject protobuf "0.6.0-beta10" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From e599bf6ecfedcc2e20df9f08b10d475d26b8dd9a Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 23 Mar 2012 13:20:02 -0700 Subject: [PATCH 227/525] Beta 10: don't dissoc out :codec_reset --- project.clj | 2 +- src/protobuf/codec.clj | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/project.clj b/project.clj index ec0a26e..365f6d4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta10" +(defproject protobuf "0.6.0-beta11" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index f1d3a73..6f3f08f 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -45,8 +45,6 @@ (if (protobuf? val) val (protobuf proto val)))))) - (gloss/compile-frame identity - #(dissoc % reset-key)) (fix repeated #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none)) From 1c7a9751024284b43d6e6f4125f5352e98f2d8da Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 26 Mar 2012 20:04:21 -0700 Subject: [PATCH 228/525] Add typehints to everything --- src/protobuf/core.clj | 12 ++++++------ src/protobuf/schema.clj | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index c127cf5..aa8334f 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -22,11 +22,11 @@ ([def] (if (or (protodef? def) (nil? def)) def - (PersistentProtocolBufferMap$Def/create - ^Descriptors.Descriptor - (if (instance? Descriptors$Descriptor def) - def - (Reflector/invokeStaticMethod ^Class def "getDescriptor" (to-array nil)))))) + (let [^Descriptors$Descriptor descriptor + (if (instance? Descriptors$Descriptor def) + def + (Reflector/invokeStaticMethod ^Class def "getDescriptor" (to-array nil)))] + (PersistentProtocolBufferMap$Def/create descriptor)))) ([def & fields] (loop [^PersistentProtocolBufferMap$Def def (protodef def) fields fields] @@ -50,7 +50,7 @@ (defn protobuf-schema "Return the schema for the given protodef." [& args] - (field-schema (.getMessageType (apply protodef args)))) + (field-schema (.getMessageType ^PersistentProtocolBufferMap$Def (apply protodef args)))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." diff --git a/src/protobuf/schema.clj b/src/protobuf/schema.clj index e9df2da..d943184 100644 --- a/src/protobuf/schema.clj +++ b/src/protobuf/schema.clj @@ -5,7 +5,7 @@ (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor Descriptors$FieldDescriptor$Type))) -(defn extension [ext field] +(defn extension [ext ^Descriptors$FieldDescriptor field] (-> (.getOptions field) (.getExtension ext) (fix string? not-empty))) @@ -13,7 +13,7 @@ (defn field-type [field] (condp instance? field Descriptors$FieldDescriptor - (if (.isRepeated field) + (if (.isRepeated ^Descriptors$FieldDescriptor field) (condp extension field (Extensions/counter) :counter (Extensions/succession) :succession @@ -27,17 +27,17 @@ (defmulti field-schema (fn [field & _] (field-type field))) -(defn struct-schema [struct & [parents]] +(defn struct-schema [^Descriptors$Descriptor struct & [parents]] (let [struct-name (.getFullName struct)] (into {:type :struct :name struct-name} (when (not-any? (partial = struct-name) parents) {:fields (into {} - (for [field (.getFields struct)] + (for [^Descriptors$FieldDescriptor field (.getFields struct)] [(PersistentProtocolBufferMap/intern (.getName field)) (field-schema field (conj parents struct-name))]))})))) -(defn basic-schema [field & [parents]] +(defn basic-schema [^Descriptors$FieldDescriptor field & [parents]] (let [java-type (keyword (lower-case (.name (.getJavaType field)))) meta-string (extension (Extensions/meta) field)] (merge (case java-type @@ -51,7 +51,7 @@ (when meta-string (read-string meta-string))))) -(defn subfield [field field-name] +(defn subfield [^Descriptors$FieldDescriptor field field-name] (.findFieldByName (.getMessageType field) (name field-name))) (defmethod field-schema :basic [field & [parents]] From 7eb40c6a768e781687ed7d346339a75dd66a152a Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 27 Mar 2012 14:12:45 -0700 Subject: [PATCH 229/525] A couple more typehints --- src/protobuf/core.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index aa8334f..b85b509 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -17,7 +17,7 @@ [obj] (instance? PersistentProtocolBufferMap$Def obj)) -(defn protodef +(defn ^PersistentProtocolBufferMap$Def protodef "Create a protodef from a string or protobuf class." ([def] (if (or (protodef? def) (nil? def)) @@ -69,7 +69,7 @@ (let [^CodedInputStream in (CodedInputStream/newInstance stream)] (PersistentProtocolBufferMap/parseFrom type in)))) -(defn protobuf-dump +(defn ^"[B" protobuf-dump "Return the byte representation of the given protobuf." ([^PersistentProtocolBufferMap p] (.toByteArray p)) From d813baaede6a4a6516a439127c8f6b4f05f0e0ef Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 27 Mar 2012 14:19:40 -0700 Subject: [PATCH 230/525] beta 12 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 365f6d4..67c5b98 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta11" +(defproject protobuf "0.6.0-beta12" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From a3508de47d46f255f58733094a72def07c2278e9 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 9 Apr 2012 13:55:56 -0700 Subject: [PATCH 231/525] add nullable enums --- proto/protobuf/core/extensions.proto | 1 + proto/protobuf/test/core.proto | 12 ++++++-- .../core/PersistentProtocolBufferMap.java | 28 ++++++++++++++++--- test/protobuf/core_test.clj | 8 ++++-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/proto/protobuf/core/extensions.proto b/proto/protobuf/core/extensions.proto index 9d61025..ce44a5a 100644 --- a/proto/protobuf/core/extensions.proto +++ b/proto/protobuf/core/extensions.proto @@ -16,4 +16,5 @@ extend google.protobuf.FieldOptions { optional sint64 null_long = 52023; optional float null_float = 52024; optional double null_double = 52025; + optional uint32 null_enum = 52026; } diff --git a/proto/protobuf/test/core.proto b/proto/protobuf/test/core.proto index c380827..480ec50 100644 --- a/proto/protobuf/test/core.proto +++ b/proto/protobuf/test/core.proto @@ -44,9 +44,17 @@ message Bar { optional float flt = 3 [(nullable) = true, (null_float) = -0.0001]; optional double dbl = 4 [(nullable) = true, (null_double) = -0.00000001]; optional string str = 5 [(nullable) = true, (null_string) = "NULL"]; + optional Enu enu = 6 [(nullable) = true, (null_enum) = 3]; - repeated string label = 6 [(nullable) = true, (null_string) = "", (succession) = true]; - repeated string labels = 7 [(nullable) = true, (null_string) = ""]; + enum Enu { + A = 0; + B = 1; + C = 2; + NULL = 3; + } + + repeated string label = 10 [(nullable) = true, (null_string) = "", (succession) = true]; + repeated string labels = 11 [(nullable) = true, (null_string) = ""]; } message Time { diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 43765a5..ce1565d 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -336,7 +336,13 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object switch (field.getJavaType()) { case ENUM: Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor) value; - return enumToKeyword(e); + if (use_extensions && + field.getOptions().getExtension(Extensions.nullable) && + field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { + return null; + } else { + return enumToKeyword(e); + } case MESSAGE: Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); DynamicMessage message = (DynamicMessage) value; @@ -348,17 +354,30 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object default: if (use_extensions && field.getOptions().getExtension(Extensions.nullable) && - field.getOptions().getExtension(nullExtension(field)).equals(value)) + field.getOptions().getExtension(nullExtension(field)).equals(value)) { return null; - return value; + } else { + return value; + } } } } static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { - if (value == null && field.getOptions().getExtension(Extensions.nullable)) + if (value == null && field.getOptions().getExtension(Extensions.nullable)) { value = field.getOptions().getExtension(nullExtension(field)); + if (field.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM) { + Descriptors.EnumDescriptor enum_type = field.getEnumType(); + Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByNumber((Integer) value); + if (enum_value == null) { + PrintWriter err = (PrintWriter) RT.ERR.deref(); + err.format("invalid enum number %s for enum type %s\n", value, enum_type.getFullName()); + } + return enum_value; + } + } + switch (field.getJavaType()) { case LONG: if (value instanceof Long) return value; @@ -405,6 +424,7 @@ static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.F case FLOAT: return Extensions.nullFloat; case DOUBLE: return Extensions.nullDouble; case STRING: return Extensions.nullString; + case ENUM: return Extensions.nullEnum; } return null; } diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index c94146f..f3d98df 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -196,20 +196,22 @@ (get-raw p :time)))))) (deftest test-nullable - (let [p (protobuf Bar :int 1 :long 330000000000 :flt 1.23 :dbl 9.87654321 :str "foo") - keyset #{:int :long :flt :dbl :str}] + (let [p (protobuf Bar :int 1 :long 330000000000 :flt 1.23 :dbl 9.87654321 :str "foo" :enu :a) + keyset #{:int :long :flt :dbl :str :enu}] (is (= 1 (get p :int))) (is (= 330000000000 (get p :long))) (is (= (float 1.23) (get p :flt))) (is (= 9.87654321 (get p :dbl))) (is (= "foo" (get p :str))) + (is (= :a (get p :enu))) (is (= keyset (set (keys p)))) - (let [p (append p {:int nil :long nil :flt nil :dbl nil :str nil})] + (let [p (append p {:int nil :long nil :flt nil :dbl nil :str nil :enu nil})] (is (= nil (get p :int))) (is (= nil (get p :long))) (is (= nil (get p :flt))) (is (= nil (get p :dbl))) (is (= nil (get p :str))) + (is (= nil (get p :enu))) (is (= keyset (set (keys p))))) (testing "nullable successions" (let [p (protobuf Bar :label "foo")] From 39c4cf3b6419203faf20bd3890a94c02bb865d33 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 9 Apr 2012 13:59:36 -0700 Subject: [PATCH 232/525] beta13 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 67c5b98..e3a7eee 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta12" +(defproject protobuf "0.6.0-beta13" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From aa82a470a3cc5d58acd7648d483a5a199953d2bb Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 9 Apr 2012 15:26:21 -0700 Subject: [PATCH 233/525] Fix some incorrect uses of transients. --- src/protobuf/core/PersistentProtocolBufferMap.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index ce1565d..19354f3 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -271,9 +271,9 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Object k = v.valAt(map_field_by); PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(k); if (existing != null) { - map.assoc(k, existing.append(v)); + map = map.assoc(k, existing.append(v)); } else { - map.assoc(k, v); + map = map.assoc(k, v); } } return map.persistent(); @@ -297,9 +297,9 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Object v = fromProtoValue(val_field, message.getField(val_field)); Object existing = map.valAt(k); if (existing != null && existing instanceof IPersistentCollection) { - map.assoc(k, ((IPersistentCollection) existing).cons(v)); + map = map.assoc(k, ((IPersistentCollection) existing).cons(v)); } else { - map.assoc(k, v); + map = map.assoc(k, v); } } return map.persistent(); @@ -315,10 +315,10 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Boolean exists = (Boolean) message.getField(exists_field); if (exists) { - set.conj(item); + set = (ITransientSet)set.conj(item); } else { try { - set.disjoin(item); + set = set.disjoin(item); } catch (Exception e) { e.printStackTrace(); } From 051482e7d28b8a8e0a9d5133a4c3363875d5c83d Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 9 Apr 2012 15:29:47 -0700 Subject: [PATCH 234/525] Redundant null check. --- src/protobuf/core/PersistentProtocolBufferMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 19354f3..927a068 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -296,7 +296,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object Object k = fromProtoValue(key_field, message.getField(key_field)); Object v = fromProtoValue(val_field, message.getField(val_field)); Object existing = map.valAt(k); - if (existing != null && existing instanceof IPersistentCollection) { + if (existing instanceof IPersistentCollection) { map = map.assoc(k, ((IPersistentCollection) existing).cons(v)); } else { map = map.assoc(k, v); From c8699d40d409cf26b635dbbfe4762f5f14e4811c Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 10 Apr 2012 16:51:13 -0700 Subject: [PATCH 235/525] add schematic dep --- project.clj | 3 ++- src/protobuf/codec.clj | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index e3a7eee..baa6547 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,8 @@ [ordered-set "0.2.3"] [useful "0.8.0-alpha1"] [fs "1.0.0"] - [conch "0.2.0"]] + [conch "0.2.0"] + [schematic "0.0.5"]] :dev-dependencies [[gloss "0.2.0-rc1"] [io "0.1.0-alpha2"]] :hooks [leiningen.protobuf] diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 6f3f08f..35419c4 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -6,6 +6,7 @@ [useful.experimental :only [lift-meta]] [clojure.java.io :only [input-stream]]) (:require io.core + [schematic.core :as schema] [gloss.core :as gloss])) (declare protobuf-codec) @@ -48,4 +49,5 @@ (fix repeated #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) :prefix :none)) - (with-meta {:schema (protobuf-schema proto)})))) + (with-meta {:schema (schema/dissoc-fields (protobuf-schema proto) + len-key reset-key)})))) From 41a32999d78f63352df3e3344ceee8ab2865c6a9 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 12 Apr 2012 16:03:32 -0700 Subject: [PATCH 236/525] Make the use_underscores "mode" per-instance. Now you can get a protobuf that does, or does not, convert _ to - without damaging anyone else using the profobuf library in your JVM. Also, stop pretending that strings and keywords are interchangeable as map keys - protobuf message fields are always keywords, and any string keys go in the extmap. --- src/protobuf/core.clj | 22 +- .../core/PersistentProtocolBufferMap.java | 192 +++++++++--------- src/protobuf/schema.clj | 54 ++--- test/protobuf/core_test.clj | 37 ++-- 4 files changed, 151 insertions(+), 154 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index b85b509..ba34d14 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -2,7 +2,7 @@ (:use [protobuf.schema :only [field-schema]] [useful.fn :only [fix]] [clojure.java.io :only [input-stream output-stream]]) - (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) + (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def PersistentProtocolBufferMap$Def$NamingStrategy Extensions) (com.google.protobuf GeneratedMessage CodedInputStream Descriptors$Descriptor) (java.io InputStream OutputStream) (clojure.lang Reflector))) @@ -20,23 +20,17 @@ (defn ^PersistentProtocolBufferMap$Def protodef "Create a protodef from a string or protobuf class." ([def] + (if (or (protodef? def) (nil? def)) + def + (protodef def PersistentProtocolBufferMap$Def/convertUnderscores))) + ([def ^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy] (if (or (protodef? def) (nil? def)) def (let [^Descriptors$Descriptor descriptor (if (instance? Descriptors$Descriptor def) def (Reflector/invokeStaticMethod ^Class def "getDescriptor" (to-array nil)))] - (PersistentProtocolBufferMap$Def/create descriptor)))) - ([def & fields] - (loop [^PersistentProtocolBufferMap$Def def (protodef def) - fields fields] - (if (empty? fields) - def - (recur (-> def - (.fieldDescriptor (first fields)) - .getMessageType - protodef) - (rest fields)))))) + (PersistentProtocolBufferMap$Def/create descriptor naming-strategy))))) (defn protobuf "Construct a protobuf of the given type." @@ -50,7 +44,8 @@ (defn protobuf-schema "Return the schema for the given protodef." [& args] - (field-schema (.getMessageType ^PersistentProtocolBufferMap$Def (apply protodef args)))) + (let [^PersistentProtocolBufferMap$Def def (apply protodef args)] + (field-schema (.getMessageType def) def))) (defn protobuf-load "Load a protobuf of the given type from an array of bytes." @@ -95,6 +90,7 @@ (.writeDelimitedTo p out)) (.flush out)))) +;; TODO make these functions nil-safe (defn append "Merge the given map into the protobuf. Equivalent to appending the byte representations." [^PersistentProtocolBufferMap p map] diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 927a068..3a7514c 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -4,7 +4,7 @@ * Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) * which can be found in the file epl-v10.html at the root of this distribution. * By using this software in any fashion, you are agreeing to be bound by - * the terms of this license. + * the terms of this license. * You must not remove this notice, or any other, from this software. **/ @@ -29,27 +29,52 @@ public class PersistentProtocolBufferMap extends APersistentMap implements IObj { public static class Def { + public static interface NamingStrategy { + String protoName(Keyword name); + Keyword clojureName(String name); + } + + public static final NamingStrategy protobufNames = new NamingStrategy() { + public String protoName(Keyword name) {return name.getName();} + public Keyword clojureName(String name) {return Keyword.intern(name.toLowerCase());} + public String toString() {return "[protobuf names]";} + }; + public static final NamingStrategy convertUnderscores = new NamingStrategy() { + public String protoName(Keyword name) {return name.getName().replaceAll("-", "_");} + public Keyword clojureName(String name) {return Keyword.intern(name.replaceAll("_", "-").toLowerCase());} + public String toString() {return "[convert underscores]";} + }; + final Descriptors.Descriptor type; + public final NamingStrategy namingStrategy; ConcurrentHashMap keyword_to_field; - static ConcurrentHashMap type_to_def = new ConcurrentHashMap(); + static ConcurrentHashMap> type_to_def = new ConcurrentHashMap>(); - public static Def create(Class c) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { + public static Def create(Class c, NamingStrategy strat) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); - return create(type); + return create(type, strat); } - public static Def create(Descriptors.Descriptor type) { - Def def = type_to_def.get(type); + public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { + ConcurrentHashMap defCache = type_to_def.get(strat); + if (defCache == null) { + defCache = new ConcurrentHashMap(); + ConcurrentHashMap previous = type_to_def.putIfAbsent(strat, defCache); + if (previous != null) defCache = previous; + } + + Def def = defCache.get(type); if (def == null) { - def = new Def(type); - type_to_def.putIfAbsent(type, def); + def = new Def(type, strat); + defCache.putIfAbsent(type, def); } return def; } - protected Def(Descriptors.Descriptor type) { + protected Def(Descriptors.Descriptor type, NamingStrategy strat) { this.type = type; this.keyword_to_field = new ConcurrentHashMap(); + this.namingStrategy = strat; } public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferException { @@ -82,13 +107,12 @@ public Descriptors.FieldDescriptor fieldDescriptor(Object key) { Keyword keyword = (Keyword) key; Descriptors.FieldDescriptor field = keyword_to_field.get(keyword); if (field == null) { - field = fieldDescriptor(keyword.getName()); + field = type.findFieldByName(namingStrategy.protoName(keyword)); if (field != null) keyword_to_field.putIfAbsent(keyword, field); } return field; } else { - String name = ((String) key).replaceAll("-","_"); - return type.findFieldByName(name); + return null; } } @@ -103,9 +127,36 @@ public String getFullName() { public Descriptors.Descriptor getMessageType() { return type; } + static final ConcurrentHashMap> caches = + new ConcurrentHashMap>(); + public Keyword intern(String name) { + ConcurrentHashMap nameCache = caches.get(namingStrategy); + if (nameCache == null) { + nameCache = new ConcurrentHashMap(); + ConcurrentHashMap existing = caches.putIfAbsent(namingStrategy, nameCache); + if (existing != null) nameCache = existing; + } + Keyword keyword = nameCache.get(name); + if (keyword == null) { + keyword = namingStrategy.clojureName(name); + Keyword existing = nameCache.putIfAbsent(name, keyword); + if (existing != null) keyword = existing; + } + return keyword; + } + + public Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { + return intern(enum_value.getName()); + } + + static final Keyword k_null = Keyword.intern(Symbol.intern("")); + protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { + Keyword keyword = intern(field.getOptions().getExtension(Extensions.mapBy)); + return keyword == k_null ? null : keyword; + } } - private final Def def; + public final Def def; private final DynamicMessage message; private final IPersistentMap _meta; private final IPersistentMap ext; @@ -203,66 +254,21 @@ protected DynamicMessage.Builder builder() { } } - static boolean use_underscores = false; - static public void setUseUnderscores(boolean val) { - use_underscores = val; - field_name_to_keyword.clear(); - enum_to_keyword.clear(); - map_field_by.clear(); - } - - static ConcurrentHashMap field_name_to_keyword = - new ConcurrentHashMap(); - static public Keyword intern(String name) { - Keyword keyword = field_name_to_keyword.get(name); - if (keyword == null) { - name = name.toLowerCase(); - if (!use_underscores) name = name.replaceAll("_","-"); - keyword = Keyword.intern(Symbol.intern(name)); - field_name_to_keyword.putIfAbsent(name, keyword); - } - return keyword; - } - - static ConcurrentHashMap enum_to_keyword = - new ConcurrentHashMap(); - static public Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { - Keyword keyword = enum_to_keyword.get(enum_value); - if (keyword == null) { - keyword = intern(enum_value.getName()); - enum_to_keyword.putIfAbsent(enum_value, keyword); - } - return keyword; - } - - static Keyword k_null = Keyword.intern(Symbol.intern("")); - static ConcurrentHashMap map_field_by = - new ConcurrentHashMap(); - static protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { - Keyword keyword = map_field_by.get(field); - if (keyword == null) { - String name = field.getOptions().getExtension(Extensions.mapBy); - keyword = intern(name); - map_field_by.putIfAbsent(field, keyword); - } - return keyword == k_null ? null : keyword; - } - - static Keyword k_key = Keyword.intern(Symbol.intern("key")); - static Keyword k_val = Keyword.intern(Symbol.intern("val")); - static Keyword k_item = Keyword.intern(Symbol.intern("item")); - static Keyword k_exists = Keyword.intern(Symbol.intern("exists")); - static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { + protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) { return fromProtoValue(field, value, true); } - static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, boolean use_extensions) { + static Keyword k_key = Keyword.intern("key"); + static Keyword k_val = Keyword.intern("val"); + static Keyword k_item = Keyword.intern("item"); + static Keyword k_exists = Keyword.intern("exists"); + protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, boolean use_extensions) { if (value instanceof List) { List values = (List) value; Iterator iterator = values.iterator(); if (use_extensions) { - Keyword map_field_by = mapFieldBy(field); + Keyword map_field_by = def.mapFieldBy(field); DescriptorProtos.FieldOptions options = field.getOptions(); if (map_field_by != null) { ITransientMap map = PersistentHashMap.EMPTY.asTransient(); @@ -286,9 +292,9 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } else if (options.getExtension(Extensions.succession)) { return fromProtoValue(field, values.get(values.size() - 1)); } else if (options.getExtension(Extensions.map)) { - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - Descriptors.FieldDescriptor key_field = def.fieldDescriptor(k_key); - Descriptors.FieldDescriptor val_field = def.fieldDescriptor(k_val); + Descriptors.Descriptor type = field.getMessageType(); + Descriptors.FieldDescriptor key_field = type.findFieldByName("key"); + Descriptors.FieldDescriptor val_field = type.findFieldByName("val"); ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { @@ -304,9 +310,9 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } return map.persistent(); } else if (options.getExtension(Extensions.set)) { - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - Descriptors.FieldDescriptor item_field = def.fieldDescriptor(k_item); - Descriptors.FieldDescriptor exists_field = def.fieldDescriptor(k_exists); + Descriptors.Descriptor type = field.getMessageType(); + Descriptors.FieldDescriptor item_field = type.findFieldByName("item"); + Descriptors.FieldDescriptor exists_field = type.findFieldByName("exists"); ITransientSet set = (ITransientSet) OrderedSet.EMPTY.asTransient(); while (iterator.hasNext()) { @@ -341,16 +347,16 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { return null; } else { - return enumToKeyword(e); + return def.enumToKeyword(e); } case MESSAGE: - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); + Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); DynamicMessage message = (DynamicMessage) value; // Total hack because getField() doesn't return an empty array for repeated messages. if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList(), use_extensions); - return new PersistentProtocolBufferMap(null, def, message); + return new PersistentProtocolBufferMap(null, fieldDef, message); default: if (use_extensions && field.getOptions().getExtension(Extensions.nullable) && @@ -363,7 +369,7 @@ static protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object } } - static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { + protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { if (value == null && field.getOptions().getExtension(Extensions.nullable)) { value = field.getOptions().getExtension(nullExtension(field)); @@ -408,8 +414,8 @@ static protected Object toProtoValue(Descriptors.FieldDescriptor field, Object v if (value instanceof PersistentProtocolBufferMap) { protobuf = (PersistentProtocolBufferMap) value; } else { - Def def = PersistentProtocolBufferMap.Def.create(field.getMessageType()); - protobuf = PersistentProtocolBufferMap.construct(def, (IPersistentMap) value); + Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); + protobuf = PersistentProtocolBufferMap.construct(fieldDef, (IPersistentMap) value); } return protobuf.message(); default: @@ -466,40 +472,36 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object addRepeatedField(builder,field, v); } } else { - Keyword map_field_by = mapFieldBy(field); + Keyword map_field_by = def.mapFieldBy(field); if (map_field_by != null) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); IPersistentMap map = (IPersistentMap) e.getValue(); Object k = e.getKey(); - Object v = toProtoValue(field, map.assoc(map_field_by, k).assoc(map_field_by.getName(), k)); + Object v = toProtoValue(field, map.assoc(map_field_by, k)); addRepeatedField(builder, field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; - Object v = toProtoValue(field, new PersistentArrayMap(map)); - addRepeatedField(builder, field, v); + addRepeatedField(builder, field, toProtoValue(field, new PersistentArrayMap(map))); } } else if (set) { - if (value instanceof IPersistentMap) { - for (ISeq s = RT.seq(value); s != null; s = s.next()) { + Object k, v; + boolean isMap = (value instanceof IPersistentMap); + for (ISeq s = RT.seq(value); s != null; s = s.next()) { + if (isMap) { Map.Entry e = (Map.Entry) s.first(); - Object[] map = {k_item, e.getKey(), k_exists, e.getValue()}; - Object v = toProtoValue(field, new PersistentArrayMap(map)); - addRepeatedField(builder, field, v); - } - } else { - for (ISeq s = RT.seq(value); s != null; s = s.next()) { - Object[] map = {k_item, s.first(), k_exists, true}; - Object v = toProtoValue(field, new PersistentArrayMap(map)); - addRepeatedField(builder, field, v); + k = e.getKey(); v = e.getValue(); + } else { + k = s.first(); v = true; } + Object[] map = {k_item, k, k_exists, v}; + addRepeatedField(builder, field, toProtoValue(field, new PersistentArrayMap(map))); } } else { - Object v = toProtoValue(field, value); - addRepeatedField(builder, field, v); + addRepeatedField(builder, field, toProtoValue(field, value)); } } } else { @@ -656,7 +658,7 @@ private static class Seq extends ASeq { public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ for (ISeq s = fields; s != null; s = s.next()) { Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); - Keyword k = intern(field.getName()); + Keyword k = proto.def.intern(field.getName()); Object v = proto.valAt(k, sentinel); if (v != sentinel) return new Seq(meta, proto, new MapEntry(k, v), s); } diff --git a/src/protobuf/schema.clj b/src/protobuf/schema.clj index d943184..a1442ac 100644 --- a/src/protobuf/schema.clj +++ b/src/protobuf/schema.clj @@ -25,25 +25,29 @@ Descriptors$Descriptor :struct)) -(defmulti field-schema (fn [field & _] (field-type field))) +(defmulti field-schema (fn [field def & _] (field-type field))) -(defn struct-schema [^Descriptors$Descriptor struct & [parents]] +(defn struct-schema [^Descriptors$Descriptor struct + ^PersistentProtocolBufferMap$Def def + & [parents]] (let [struct-name (.getFullName struct)] (into {:type :struct :name struct-name} (when (not-any? (partial = struct-name) parents) {:fields (into {} (for [^Descriptors$FieldDescriptor field (.getFields struct)] - [(PersistentProtocolBufferMap/intern (.getName field)) - (field-schema field (conj parents struct-name))]))})))) + [(.intern def (.getName field)) + (field-schema field def (conj parents struct-name))]))})))) -(defn basic-schema [^Descriptors$FieldDescriptor field & [parents]] +(defn basic-schema [^Descriptors$FieldDescriptor field + ^PersistentProtocolBufferMap$Def def + & [parents]] (let [java-type (keyword (lower-case (.name (.getJavaType field)))) meta-string (extension (Extensions/meta) field)] (merge (case java-type - :message (struct-schema (.getMessageType field) parents) + :message (struct-schema (.getMessageType field) def parents) :enum {:type :enum - :values (set (map #(PersistentProtocolBufferMap/enumToKeyword %) + :values (set (map #(.enumToKeyword def %) (.. field getEnumType getValues)))} {:type java-type}) (when (.hasDefaultValue field) @@ -54,35 +58,35 @@ (defn subfield [^Descriptors$FieldDescriptor field field-name] (.findFieldByName (.getMessageType field) (name field-name))) -(defmethod field-schema :basic [field & [parents]] - (basic-schema field parents)) +(defmethod field-schema :basic [field def & [parents]] + (basic-schema field def parents)) -(defmethod field-schema :list [field & [parents]] +(defmethod field-schema :list [field def & [parents]] {:type :list - :values (basic-schema field parents)}) + :values (basic-schema field def parents)}) -(defmethod field-schema :succession [field & [parents]] - (assoc (basic-schema field parents) +(defmethod field-schema :succession [field def & [parents]] + (assoc (basic-schema field def parents) :succession true)) -(defmethod field-schema :counter [field & [parents]] - (assoc (basic-schema field parents) +(defmethod field-schema :counter [field def & [parents]] + (assoc (basic-schema field def parents) :counter true)) -(defmethod field-schema :set [field & [parents]] +(defmethod field-schema :set [field def & [parents]] {:type :set - :values (field-schema (subfield field :item) parents)}) + :values (field-schema (subfield field :item) def parents)}) -(defmethod field-schema :map [field & [parents]] +(defmethod field-schema :map [field def & [parents]] {:type :map - :keys (field-schema (subfield field :key) parents) - :values (field-schema (subfield field :val) parents)}) + :keys (field-schema (subfield field :key) def parents) + :values (field-schema (subfield field :val) def parents)}) -(defmethod field-schema :map-by [field & [parents]] +(defmethod field-schema :map-by [field def & [parents]] (let [map-by (extension (Extensions/mapBy) field)] {:type :map - :keys (field-schema (subfield field map-by) parents) - :values (basic-schema field parents)})) + :keys (field-schema (subfield field map-by) def parents) + :values (basic-schema field def parents)})) -(defmethod field-schema :struct [field & [parents]] - (struct-schema field parents)) +(defmethod field-schema :struct [field def & [parents]] + (struct-schema field def parents)) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index f3d98df..b4424ec 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -3,6 +3,8 @@ (:import (java.io PipedInputStream PipedOutputStream))) (def Foo (protodef protobuf.test.Core$Foo)) +(def FooUnder (protodef protobuf.test.Core$Foo + protobuf.core.PersistentProtocolBufferMap$Def/protobufNames)) (def Bar (protodef protobuf.test.Core$Bar)) (def Response (protodef protobuf.test.Core$Response)) (def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) @@ -54,7 +56,7 @@ (let [p (assoc p :label "baz" :tags ["nuprin"])] (is (= ["nuprin"] (:tags p))) (is (= "baz" (:label p)))) - (let [p (assoc p "responses" [:yes :no :maybe :no "yes"])] + (let [p (assoc p :responses [:yes :no :maybe :no "yes"])] (is (= [:yes :no :maybe :no :yes] (:responses p)))) (let [p (assoc p :tags "aspirin")] (is (= ["aspirin"] (:tags p)))) @@ -96,15 +98,13 @@ (is (= (into {} m2) (into {} p2))) (is (= (set (keys m2)) (set (keys p2)))))) -(deftest test-string-keys - (let [p (protobuf Foo "id" 5 "label" "rad")] +(deftest test-string-keys-in-extmap + (let [p (protobuf Foo :id 5 :label "red") + q (assoc p "label" "blue")] (is (= 5 (p :id))) - (is (= 5 (p "id"))) - (is (= "rad" (p :label))) - (is (= "rad" (p "label"))) - (let [p (conj p {"tags" ["check" "it" "out"]})] - (is (= ["check" "it" "out"] (p :tags))) - (is (= ["check" "it" "out"] (p "tags")))))) + (is (not (contains? p "id"))) + (is (= "red" (q :label))) + (is (= "blue" (q "label"))))) (deftest test-append-bytes (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) @@ -150,9 +150,6 @@ (is (= "bar" (get-in p [:item-map "bar" :item]))))) (deftest test-map-by-with-inconsistent-keys - (let [p (protobuf Foo :pair-map {"foo" {"key" "bar" "val" "hmm"}})] - (is (= "hmm" (get-in p [:pair-map "foo" :val]))) - (is (= nil (get-in p [:pair-map "bar" :val])))) (let [p (protobuf Foo :pair-map {"foo" {:key "bar" :val "hmm"}})] (is (= "hmm" (get-in p [:pair-map "foo" :val]))) (is (= nil (get-in p [:pair-map "bar" :val]))))) @@ -293,19 +290,17 @@ (is (= {} (default-protobuf protobuf.test.Core$Foo :groups)))) (deftest test-use-underscores - (let [p (protobuf Foo {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]})] - (is (= '(:id :responses :tag-set :deleted) (keys p))) - (is (= [:yes :not-sure :maybe :not-sure :no] (:responses p))) + (let [[dashes underscores] (for [proto [Foo FooUnder]] + (protobuf proto {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]}))] + (is (= '(:id :responses :tag-set :deleted) (keys dashes))) + (is (= [:yes :not-sure :maybe :not-sure :no] (:responses dashes))) - (protobuf.core.PersistentProtocolBufferMap/setUseUnderscores true) - (is (= '(:id :responses :tag_set :deleted) (keys p))) - (is (= [:yes :not_sure :maybe :not_sure :no] (:responses p))) + (is (= '(:id :responses :tag_set :deleted) (keys underscores))) + (is (= [:yes :not_sure :maybe :not_sure :no] (:responses underscores))) (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id :pair_map :groups :doubles :floats :item_map :counts :time :lat :long} - (-> (protobuf-schema Foo) :fields keys set))) - - (protobuf.core.PersistentProtocolBufferMap/setUseUnderscores false))) + (-> (protobuf-schema FooUnder) :fields keys set))))) (deftest test-protobuf-nested-message (let [p (protobuf Response :ok false :error (protobuf ErrorMsg :code -10 :data "abc"))] From 923fda6364303beba4c501fbd2554d138625dc25 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 13 Apr 2012 13:20:35 -0700 Subject: [PATCH 237/525] Go back to treating strings and keywords the same. Also, no longer automatically convert enum names to and from uppercase. Just use a proto that has lower-case enum names if you want lower-case keywords. --- proto/protobuf/test/core.proto | 16 +-- src/protobuf/core.clj | 2 +- .../core/PersistentProtocolBufferMap.java | 113 +++++++++++------- src/protobuf/schema.clj | 2 +- test/protobuf/core_test.clj | 23 ++-- 5 files changed, 93 insertions(+), 63 deletions(-) diff --git a/proto/protobuf/test/core.proto b/proto/protobuf/test/core.proto index 480ec50..0bb5612 100644 --- a/proto/protobuf/test/core.proto +++ b/proto/protobuf/test/core.proto @@ -23,10 +23,10 @@ message Foo { repeated Time time = 11 [(succession) = true]; enum Response { - YES = 0; - NO = 1; - MAYBE = 2; - NOT_SURE = 3; + yes = 0; + no = 1; + maybe = 2; + not_sure = 3; } repeated Item tag_set = 20 [(set) = true]; @@ -47,10 +47,10 @@ message Bar { optional Enu enu = 6 [(nullable) = true, (null_enum) = 3]; enum Enu { - A = 0; - B = 1; - C = 2; - NULL = 3; + a = 0; + b = 1; + c = 2; + nil = 3; } repeated string label = 10 [(nullable) = true, (null_string) = "", (succession) = true]; diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index ba34d14..e47d2ea 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -24,7 +24,7 @@ def (protodef def PersistentProtocolBufferMap$Def/convertUnderscores))) ([def ^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy] - (if (or (protodef? def) (nil? def)) + (if (nil? def) def (let [^Descriptors$Descriptor descriptor (if (instance? Descriptors$Descriptor def) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 3a7514c..d48dd31 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -30,24 +30,45 @@ public class PersistentProtocolBufferMap extends APersistentMap implements IObj { public static class Def { public static interface NamingStrategy { - String protoName(Keyword name); - Keyword clojureName(String name); + /** + * Given a Clojure map key, return the string to be used as the protobuf message field name. + */ + String protoName(Object clojureName); + + /** + * Given a protobuf message field name, return a Clojure object suitable for use as a map key. + */ + Object clojureName(String protoName); + } + + // we want this to work for anything Named, so use clojure.core/name + public static final Var NAME_VAR = Var.intern(RT.CLOJURE_NS, Symbol.intern("name")); + public static final String nameStr(Object named) { + try { + return (String)((IFn)NAME_VAR.deref()).invoke(named); + } catch (Exception e) { + return null; + } } public static final NamingStrategy protobufNames = new NamingStrategy() { - public String protoName(Keyword name) {return name.getName();} - public Keyword clojureName(String name) {return Keyword.intern(name.toLowerCase());} + public String protoName(Object name) {return nameStr(name);} + public Object clojureName(String name) {return Keyword.intern(name.toLowerCase());} public String toString() {return "[protobuf names]";} }; public static final NamingStrategy convertUnderscores = new NamingStrategy() { - public String protoName(Keyword name) {return name.getName().replaceAll("-", "_");} - public Keyword clojureName(String name) {return Keyword.intern(name.replaceAll("_", "-").toLowerCase());} + public String protoName(Object name) {return nameStr(name).replaceAll("-", "_");} + public Object clojureName(String name) {return Keyword.intern(name.replaceAll("_", "-").toLowerCase());} public String toString() {return "[convert underscores]";} }; final Descriptors.Descriptor type; public final NamingStrategy namingStrategy; - ConcurrentHashMap keyword_to_field; + + public static final Object NULL = new Object(); + // keys should be FieldDescriptors, except that NULL is used as a replacement for real null + ConcurrentHashMap key_to_field; + static ConcurrentHashMap> type_to_def = new ConcurrentHashMap>(); public static Def create(Class c, NamingStrategy strat) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { @@ -72,9 +93,9 @@ public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { } protected Def(Descriptors.Descriptor type, NamingStrategy strat) { - this.type = type; - this.keyword_to_field = new ConcurrentHashMap(); - this.namingStrategy = strat; + this.type = type; + this.key_to_field = new ConcurrentHashMap(); + this.namingStrategy = strat; } public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferException { @@ -102,17 +123,20 @@ public Descriptors.FieldDescriptor fieldDescriptor(Object key) { if (key == null) return null; if (key instanceof Descriptors.FieldDescriptor) { - return (Descriptors.FieldDescriptor) key; - } else if (key instanceof Keyword) { - Keyword keyword = (Keyword) key; - Descriptors.FieldDescriptor field = keyword_to_field.get(keyword); - if (field == null) { - field = type.findFieldByName(namingStrategy.protoName(keyword)); - if (field != null) keyword_to_field.putIfAbsent(keyword, field); - } - return field; + return (Descriptors.FieldDescriptor)key; } else { - return null; + Object field = key_to_field.get(key); + if (field != null) { + if (field == NULL) { + return null; + } + return (Descriptors.FieldDescriptor)field; + } + else { + field = type.findFieldByName(namingStrategy.protoName(key)); + key_to_field.putIfAbsent(key, field == null ? NULL : field); + } + return (Descriptors.FieldDescriptor)field; } } @@ -127,32 +151,32 @@ public String getFullName() { public Descriptors.Descriptor getMessageType() { return type; } - static final ConcurrentHashMap> caches = - new ConcurrentHashMap>(); - public Keyword intern(String name) { - ConcurrentHashMap nameCache = caches.get(namingStrategy); + static final ConcurrentHashMap> caches = + new ConcurrentHashMap>(); + public Object intern(String name) { + ConcurrentHashMap nameCache = caches.get(namingStrategy); if (nameCache == null) { - nameCache = new ConcurrentHashMap(); - ConcurrentHashMap existing = caches.putIfAbsent(namingStrategy, nameCache); + nameCache = new ConcurrentHashMap(); + ConcurrentHashMap existing = caches.putIfAbsent(namingStrategy, nameCache); if (existing != null) nameCache = existing; } - Keyword keyword = nameCache.get(name); - if (keyword == null) { - keyword = namingStrategy.clojureName(name); - Keyword existing = nameCache.putIfAbsent(name, keyword); - if (existing != null) keyword = existing; + Object clojureName = nameCache.get(name); + if (clojureName == null) { + clojureName = namingStrategy.clojureName(name); + Object existing = nameCache.putIfAbsent(name, clojureName); + if (existing != null) clojureName = existing; } - return keyword; + return clojureName; } - public Keyword enumToKeyword(Descriptors.EnumValueDescriptor enum_value) { + public Object clojureEnumValue(Descriptors.EnumValueDescriptor enum_value) { return intern(enum_value.getName()); } - static final Keyword k_null = Keyword.intern(Symbol.intern("")); - protected Keyword mapFieldBy(Descriptors.FieldDescriptor field) { - Keyword keyword = intern(field.getOptions().getExtension(Extensions.mapBy)); - return keyword == k_null ? null : keyword; + static final Object k_null = Keyword.intern(Symbol.intern("")); + protected Object mapFieldBy(Descriptors.FieldDescriptor field) { + Object key = intern(field.getOptions().getExtension(Extensions.mapBy)); + return key == k_null ? null : key; } } @@ -268,7 +292,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, Iterator iterator = values.iterator(); if (use_extensions) { - Keyword map_field_by = def.mapFieldBy(field); + Object map_field_by = def.mapFieldBy(field); DescriptorProtos.FieldOptions options = field.getOptions(); if (map_field_by != null) { ITransientMap map = PersistentHashMap.EMPTY.asTransient(); @@ -347,7 +371,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { return null; } else { - return def.enumToKeyword(e); + return def.clojureEnumValue(e); } case MESSAGE: Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); @@ -400,8 +424,7 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { if (value instanceof Float) return new Double((Float) value); return value; case ENUM: - String name = (value instanceof Keyword) ? ((Keyword) value).getName() : (String) value; - name = name.toUpperCase().replaceAll("-","_"); + String name = def.namingStrategy.protoName(value); Descriptors.EnumDescriptor enum_type = field.getEnumType(); Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); if (enum_value == null) { @@ -438,7 +461,7 @@ static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.F protected void addRepeatedField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { try { builder.addRepeatedField(field, value); - } catch (IllegalArgumentException e) { + } catch (Exception e) { String msg = String.format("error adding %s to %s field %s", value, field.getJavaType().toString().toLowerCase(), field.getFullName()); @@ -469,10 +492,10 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object if (value instanceof Sequential && !set) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Object v = toProtoValue(field, s.first()); - addRepeatedField(builder,field, v); + addRepeatedField(builder, field, v); } } else { - Keyword map_field_by = def.mapFieldBy(field); + Object map_field_by = def.mapFieldBy(field); if (map_field_by != null) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry) s.first(); @@ -658,7 +681,7 @@ private static class Seq extends ASeq { public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ for (ISeq s = fields; s != null; s = s.next()) { Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); - Keyword k = proto.def.intern(field.getName()); + Object k = proto.def.intern(field.getName()); Object v = proto.valAt(k, sentinel); if (v != sentinel) return new Seq(meta, proto, new MapEntry(k, v), s); } diff --git a/src/protobuf/schema.clj b/src/protobuf/schema.clj index a1442ac..1702c9c 100644 --- a/src/protobuf/schema.clj +++ b/src/protobuf/schema.clj @@ -47,7 +47,7 @@ (merge (case java-type :message (struct-schema (.getMessageType field) def parents) :enum {:type :enum - :values (set (map #(.enumToKeyword def %) + :values (set (map #(.clojureEnumValue def %) (.. field getEnumType getValues)))} {:type java-type}) (when (.hasDefaultValue field) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index b4424ec..0f4fa5f 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -98,13 +98,15 @@ (is (= (into {} m2) (into {} p2))) (is (= (set (keys m2)) (set (keys p2)))))) -(deftest test-string-keys-in-extmap - (let [p (protobuf Foo :id 5 :label "red") - q (assoc p "label" "blue")] +(deftest test-string-keys + (let [p (protobuf Foo "id" 5 "label" "rad")] (is (= 5 (p :id))) - (is (not (contains? p "id"))) - (is (= "red" (q :label))) - (is (= "blue" (q "label"))))) + (is (= 5 (p "id"))) + (is (= "rad" (p :label))) + (is (= "rad" (p "label"))) + (let [p (conj p {"tags" ["check" "it" "out"]})] + (is (= ["check" "it" "out"] (p :tags))) + (is (= ["check" "it" "out"] (p "tags")))))) (deftest test-append-bytes (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) @@ -150,6 +152,9 @@ (is (= "bar" (get-in p [:item-map "bar" :item]))))) (deftest test-map-by-with-inconsistent-keys + (let [p (protobuf Foo :pair-map {"foo" {"key" "bar" "val" "hmm"}})] + (is (= "hmm" (get-in p [:pair-map "foo" :val]))) + (is (= nil (get-in p [:pair-map "bar" :val])))) (let [p (protobuf Foo :pair-map {"foo" {:key "bar" :val "hmm"}})] (is (= "hmm" (get-in p [:pair-map "foo" :val]))) (is (= nil (get-in p [:pair-map "bar" :val]))))) @@ -290,8 +295,10 @@ (is (= {} (default-protobuf protobuf.test.Core$Foo :groups)))) (deftest test-use-underscores - (let [[dashes underscores] (for [proto [Foo FooUnder]] - (protobuf proto {:tag_set ["odd"] :responses [:yes :not-sure :maybe :not-sure :no]}))] + (let [dashes (protobuf Foo {:tag_set ["odd"] + :responses [:yes :not-sure :maybe :not-sure :no]}) + underscores (protobuf FooUnder {:tag_set ["odd"] + :responses [:yes :not_sure :maybe :not_sure :no]})] (is (= '(:id :responses :tag-set :deleted) (keys dashes))) (is (= [:yes :not-sure :maybe :not-sure :no] (:responses dashes))) From 43db0bcfe09452a9b9d9d58cd6e55cb3dfb7b54a Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 13 Apr 2012 16:35:37 -0700 Subject: [PATCH 238/525] beta 14 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index baa6547..b24fd47 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta13" +(defproject protobuf "0.6.0-beta14" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] [ordered-set "0.2.3"] From 1b9299ea7970ac64b78eb051493b24b59ed87bcf Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 14:19:05 -0700 Subject: [PATCH 239/525] Replace uses of raw types with generics --- .../core/PersistentProtocolBufferMap.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index d48dd31..b416b1f 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.lang.reflect.InvocationTargetException; +import com.google.protobuf.DescriptorProtos.FieldOptions; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Descriptors; @@ -288,8 +289,8 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) static Keyword k_exists = Keyword.intern("exists"); protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, boolean use_extensions) { if (value instanceof List) { - List values = (List) value; - Iterator iterator = values.iterator(); + List values = (List) value; + Iterator iterator = values.iterator(); if (use_extensions) { Object map_field_by = def.mapFieldBy(field); @@ -378,7 +379,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, DynamicMessage message = (DynamicMessage) value; // Total hack because getField() doesn't return an empty array for repeated messages. - if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList(), use_extensions); + if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList(), use_extensions); return new PersistentProtocolBufferMap(null, fieldDef, message); default: @@ -446,7 +447,7 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { } } - static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.FieldDescriptor field) { + static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.FieldDescriptor field) { switch (field.getJavaType()) { case LONG: return Extensions.nullLong; case INT: return Extensions.nullInt; @@ -498,7 +499,7 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object Object map_field_by = def.mapFieldBy(field); if (map_field_by != null) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry) s.first(); IPersistentMap map = (IPersistentMap) e.getValue(); Object k = e.getKey(); Object v = toProtoValue(field, map.assoc(map_field_by, k)); @@ -506,7 +507,7 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object } } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry) s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; addRepeatedField(builder, field, toProtoValue(field, new PersistentArrayMap(map))); } @@ -515,7 +516,7 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object boolean isMap = (value instanceof IPersistentMap); for (ISeq s = RT.seq(value); s != null; s = s.next()) { if (isMap) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry) s.first(); k = e.getKey(); v = e.getValue(); } else { k = s.first(); v = true; @@ -611,7 +612,7 @@ public IPersistentMap assocEx(Object key, Object value) throws Exception { public IPersistentCollection cons(Object o) { DynamicMessage.Builder builder = builder(); if (o instanceof Map.Entry) { - Map.Entry e = (Map.Entry) o; + Map.Entry e = (Map.Entry) o; addField(builder, e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector) o; @@ -619,7 +620,7 @@ public IPersistentCollection cons(Object o) { addField(builder, v.nth(0), v.nth(1)); } else { for (ISeq s = RT.seq(o); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry) s.first(); addField(builder, e.getKey(), e.getValue()); } } @@ -650,7 +651,7 @@ public IPersistentMap without(Object key) throws Exception { return new PersistentProtocolBufferMap(meta(), ext, def, builder().clearField(field)); } - public Iterator iterator() { + public Iterator iterator() { return new SeqIterator(seq()); } From 5e7f7e422fe873cb8f20e224d4573ef8d5509e17 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 14:19:16 -0700 Subject: [PATCH 240/525] Remove dead code --- src/protobuf/core/PersistentProtocolBufferMap.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index b416b1f..bd17407 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -186,8 +186,6 @@ protected Object mapFieldBy(Descriptors.FieldDescriptor field) { private final IPersistentMap _meta; private final IPersistentMap ext; - private DynamicMessage built_message; - static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws InvalidProtocolBufferException { DynamicMessage message = def.parseFrom(bytes); return new PersistentProtocolBufferMap(null, def, message); @@ -656,7 +654,6 @@ public Iterator iterator() { } public int count() { - DynamicMessage message = message(); int count = RT.count(ext); for (Descriptors.FieldDescriptor field : def.type.getFields()) { if (protoContainsKey(field)) { From aa2444031b4e233e6799aa89cb94c8a674596a2e Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 14:19:34 -0700 Subject: [PATCH 241/525] Fold default into switch/case --- src/protobuf/core/PersistentProtocolBufferMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index bd17407..e4b2b46 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -453,8 +453,8 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { case DOUBLE: return Extensions.nullDouble; case STRING: return Extensions.nullString; case ENUM: return Extensions.nullEnum; + default: return null; } - return null; } protected void addRepeatedField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { From 0aa36f6ad0585263139a41d673772a7eda2f9f7d Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 14:56:55 -0700 Subject: [PATCH 242/525] Regularize whitespace with Eclipse --- .../core/PersistentProtocolBufferMap.java | 391 ++++++++++-------- 1 file changed, 225 insertions(+), 166 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index e4b2b46..d61549d 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -28,6 +28,7 @@ import com.google.protobuf.CodedOutputStream; import com.google.protobuf.GeneratedMessage; + public class PersistentProtocolBufferMap extends APersistentMap implements IObj { public static class Def { public static interface NamingStrategy { @@ -35,7 +36,7 @@ public static interface NamingStrategy { * Given a Clojure map key, return the string to be used as the protobuf message field name. */ String protoName(Object clojureName); - + /** * Given a protobuf message field name, return a Clojure object suitable for use as a map key. */ @@ -44,6 +45,7 @@ public static interface NamingStrategy { // we want this to work for anything Named, so use clojure.core/name public static final Var NAME_VAR = Var.intern(RT.CLOJURE_NS, Symbol.intern("name")); + public static final String nameStr(Object named) { try { return (String)((IFn)NAME_VAR.deref()).invoke(named); @@ -53,15 +55,31 @@ public static final String nameStr(Object named) { } public static final NamingStrategy protobufNames = new NamingStrategy() { - public String protoName(Object name) {return nameStr(name);} - public Object clojureName(String name) {return Keyword.intern(name.toLowerCase());} - public String toString() {return "[protobuf names]";} - }; + public String protoName(Object name) { + return nameStr(name); + } + + public Object clojureName(String name) { + return Keyword.intern(name.toLowerCase()); + } + + public String toString() { + return "[protobuf names]"; + } + }; public static final NamingStrategy convertUnderscores = new NamingStrategy() { - public String protoName(Object name) {return nameStr(name).replaceAll("-", "_");} - public Object clojureName(String name) {return Keyword.intern(name.replaceAll("_", "-").toLowerCase());} - public String toString() {return "[convert underscores]";} - }; + public String protoName(Object name) { + return nameStr(name).replaceAll("-", "_"); + } + + public Object clojureName(String name) { + return Keyword.intern(name.replaceAll("_", "-").toLowerCase()); + } + + public String toString() { + return "[convert underscores]"; + } + }; final Descriptors.Descriptor type; public final NamingStrategy namingStrategy; @@ -69,11 +87,13 @@ public static final String nameStr(Object named) { public static final Object NULL = new Object(); // keys should be FieldDescriptors, except that NULL is used as a replacement for real null ConcurrentHashMap key_to_field; - + static ConcurrentHashMap> type_to_def = new ConcurrentHashMap>(); - public static Def create(Class c, NamingStrategy strat) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - Descriptors.Descriptor type = (Descriptors.Descriptor) c.getMethod("getDescriptor").invoke(null); + public static Def create(Class c, NamingStrategy strat) throws NoSuchMethodException, + IllegalAccessException, InvocationTargetException { + Descriptors.Descriptor type = (Descriptors.Descriptor)c.getMethod("getDescriptor").invoke( + null); return create(type, strat); } @@ -81,8 +101,10 @@ public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { ConcurrentHashMap defCache = type_to_def.get(strat); if (defCache == null) { defCache = new ConcurrentHashMap(); - ConcurrentHashMap previous = type_to_def.putIfAbsent(strat, defCache); - if (previous != null) defCache = previous; + ConcurrentHashMap previous = type_to_def.putIfAbsent(strat, + defCache); + if (previous != null) + defCache = previous; } Def def = defCache.get(type); @@ -94,8 +116,8 @@ public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { } protected Def(Descriptors.Descriptor type, NamingStrategy strat) { - this.type = type; - this.key_to_field = new ConcurrentHashMap(); + this.type = type; + this.key_to_field = new ConcurrentHashMap(); this.namingStrategy = strat; } @@ -121,7 +143,8 @@ public DynamicMessage.Builder newBuilder() { } public Descriptors.FieldDescriptor fieldDescriptor(Object key) { - if (key == null) return null; + if (key == null) + return null; if (key instanceof Descriptors.FieldDescriptor) { return (Descriptors.FieldDescriptor)key; @@ -132,8 +155,7 @@ public Descriptors.FieldDescriptor fieldDescriptor(Object key) { return null; } return (Descriptors.FieldDescriptor)field; - } - else { + } else { field = type.findFieldByName(namingStrategy.protoName(key)); key_to_field.putIfAbsent(key, field == null ? NULL : field); } @@ -152,20 +174,23 @@ public String getFullName() { public Descriptors.Descriptor getMessageType() { return type; } - static final ConcurrentHashMap> caches = - new ConcurrentHashMap>(); + + static final ConcurrentHashMap> caches = new ConcurrentHashMap>(); + public Object intern(String name) { ConcurrentHashMap nameCache = caches.get(namingStrategy); if (nameCache == null) { nameCache = new ConcurrentHashMap(); ConcurrentHashMap existing = caches.putIfAbsent(namingStrategy, nameCache); - if (existing != null) nameCache = existing; + if (existing != null) + nameCache = existing; } Object clojureName = nameCache.get(name); if (clojureName == null) { clojureName = namingStrategy.clojureName(name); Object existing = nameCache.putIfAbsent(name, clojureName); - if (existing != null) clojureName = existing; + if (existing != null) + clojureName = existing; } return clojureName; } @@ -175,6 +200,7 @@ public Object clojureEnumValue(Descriptors.EnumValueDescriptor enum_value) { } static final Object k_null = Keyword.intern(Symbol.intern("")); + protected Object mapFieldBy(Descriptors.FieldDescriptor field) { Object key = intern(field.getOptions().getExtension(Extensions.mapBy)); return key == k_null ? null : key; @@ -186,17 +212,20 @@ protected Object mapFieldBy(Descriptors.FieldDescriptor field) { private final IPersistentMap _meta; private final IPersistentMap ext; - static public PersistentProtocolBufferMap create(Def def, byte[] bytes) throws InvalidProtocolBufferException { + static public PersistentProtocolBufferMap create(Def def, byte[] bytes) + throws InvalidProtocolBufferException { DynamicMessage message = def.parseFrom(bytes); return new PersistentProtocolBufferMap(null, def, message); } - static public PersistentProtocolBufferMap parseFrom(Def def, CodedInputStream input) throws IOException { + static public PersistentProtocolBufferMap parseFrom(Def def, CodedInputStream input) + throws IOException { DynamicMessage message = def.parseFrom(input); return new PersistentProtocolBufferMap(null, def, message); } - static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStream input) throws IOException { + static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStream input) + throws IOException { DynamicMessage.Builder builder = def.parseDelimitedFrom(input); if (builder != null) { return new PersistentProtocolBufferMap(null, def, builder); @@ -207,41 +236,43 @@ static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStrea static public PersistentProtocolBufferMap construct(Def def, Object keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); - return (PersistentProtocolBufferMap) protobuf.cons(keyvals); + return (PersistentProtocolBufferMap)protobuf.cons(keyvals); } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { - this._meta = meta; - this.ext = null; - this.def = def; + this._meta = meta; + this.ext = null; + this.def = def; this.message = null; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage message) { - this._meta = meta; - this.ext = null; - this.def = def; + this._meta = meta; + this.ext = null; + this.def = def; this.message = message; } - protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, DynamicMessage message) { - this._meta = meta; - this.ext = ext; - this.def = def; + protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, + DynamicMessage message) { + this._meta = meta; + this.ext = ext; + this.def = def; this.message = message; } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def, DynamicMessage.Builder builder) { - this._meta = meta; - this.ext = null; - this.def = def; + this._meta = meta; + this.ext = null; + this.def = def; this.message = builder.build(); } - protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, DynamicMessage.Builder builder) { - this._meta = meta; - this.ext = ext; - this.def = def; + protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, Def def, + DynamicMessage.Builder builder) { + this._meta = meta; + this.ext = ext; + this.def = def; this.message = builder.build(); } @@ -281,13 +312,15 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) return fromProtoValue(field, value, true); } - static Keyword k_key = Keyword.intern("key"); - static Keyword k_val = Keyword.intern("val"); - static Keyword k_item = Keyword.intern("item"); + static Keyword k_key = Keyword.intern("key"); + static Keyword k_val = Keyword.intern("val"); + static Keyword k_item = Keyword.intern("item"); static Keyword k_exists = Keyword.intern("exists"); - protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, boolean use_extensions) { + + protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, + boolean use_extensions) { if (value instanceof List) { - List values = (List) value; + List values = (List)value; Iterator iterator = values.iterator(); if (use_extensions) { @@ -296,9 +329,10 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, if (map_field_by != null) { ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { - PersistentProtocolBufferMap v = (PersistentProtocolBufferMap) fromProtoValue(field, iterator.next()); + PersistentProtocolBufferMap v = (PersistentProtocolBufferMap)fromProtoValue(field, + iterator.next()); Object k = v.valAt(map_field_by); - PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap) map.valAt(k); + PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap)map.valAt(k); if (existing != null) { map = map.assoc(k, existing.append(v)); } else { @@ -321,12 +355,12 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, ITransientMap map = PersistentHashMap.EMPTY.asTransient(); while (iterator.hasNext()) { - DynamicMessage message = (DynamicMessage) iterator.next(); + DynamicMessage message = (DynamicMessage)iterator.next(); Object k = fromProtoValue(key_field, message.getField(key_field)); Object v = fromProtoValue(val_field, message.getField(val_field)); Object existing = map.valAt(k); if (existing instanceof IPersistentCollection) { - map = map.assoc(k, ((IPersistentCollection) existing).cons(v)); + map = map.assoc(k, ((IPersistentCollection)existing).cons(v)); } else { map = map.assoc(k, v); } @@ -337,11 +371,11 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, Descriptors.FieldDescriptor item_field = type.findFieldByName("item"); Descriptors.FieldDescriptor exists_field = type.findFieldByName("exists"); - ITransientSet set = (ITransientSet) OrderedSet.EMPTY.asTransient(); + ITransientSet set = (ITransientSet)OrderedSet.EMPTY.asTransient(); while (iterator.hasNext()) { - DynamicMessage message = (DynamicMessage) iterator.next(); - Object item = fromProtoValue(item_field, message.getField(item_field)); - Boolean exists = (Boolean) message.getField(exists_field); + DynamicMessage message = (DynamicMessage)iterator.next(); + Object item = fromProtoValue(item_field, message.getField(item_field)); + Boolean exists = (Boolean)message.getField(exists_field); if (exists) { set = (ITransientSet)set.conj(item); @@ -363,31 +397,31 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, return PersistentVector.create(list); } else { switch (field.getJavaType()) { - case ENUM: - Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor) value; - if (use_extensions && - field.getOptions().getExtension(Extensions.nullable) && - field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { - return null; - } else { - return def.clojureEnumValue(e); - } - case MESSAGE: - Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); - DynamicMessage message = (DynamicMessage) value; - - // Total hack because getField() doesn't return an empty array for repeated messages. - if (field.isRepeated() && !message.isInitialized()) return fromProtoValue(field, new ArrayList(), use_extensions); - - return new PersistentProtocolBufferMap(null, fieldDef, message); - default: - if (use_extensions && - field.getOptions().getExtension(Extensions.nullable) && - field.getOptions().getExtension(nullExtension(field)).equals(value)) { - return null; - } else { - return value; - } + case ENUM: + Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor)value; + if (use_extensions && field.getOptions().getExtension(Extensions.nullable) + && field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { + return null; + } else { + return def.clojureEnumValue(e); + } + case MESSAGE: + Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), + this.def.namingStrategy); + DynamicMessage message = (DynamicMessage)value; + + // Total hack because getField() doesn't return an empty array for repeated messages. + if (field.isRepeated() && !message.isInitialized()) + return fromProtoValue(field, new ArrayList(), use_extensions); + + return new PersistentProtocolBufferMap(null, fieldDef, message); + default: + if (use_extensions && field.getOptions().getExtension(Extensions.nullable) + && field.getOptions().getExtension(nullExtension(field)).equals(value)) { + return null; + } else { + return value; + } } } } @@ -397,10 +431,10 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { value = field.getOptions().getExtension(nullExtension(field)); if (field.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM) { - Descriptors.EnumDescriptor enum_type = field.getEnumType(); - Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByNumber((Integer) value); + Descriptors.EnumDescriptor enum_type = field.getEnumType(); + Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByNumber((Integer)value); if (enum_value == null) { - PrintWriter err = (PrintWriter) RT.ERR.deref(); + PrintWriter err = (PrintWriter)RT.ERR.deref(); err.format("invalid enum number %s for enum type %s\n", value, enum_type.getFullName()); } return enum_value; @@ -408,82 +442,100 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { } switch (field.getJavaType()) { - case LONG: - if (value instanceof Long) return value; - return new Long(((Integer) value).longValue()); - case INT: - if (value instanceof Integer) return value; - return new Integer(((Long) value).intValue()); - case FLOAT: - if (value instanceof Integer) return new Float((Integer) value * 1.0); - if (value instanceof Double) return new Float((Double) value); - return value; - case DOUBLE: - if (value instanceof Integer) return new Double((Integer) value * 1.0); - if (value instanceof Float) return new Double((Float) value); - return value; - case ENUM: - String name = def.namingStrategy.protoName(value); - Descriptors.EnumDescriptor enum_type = field.getEnumType(); - Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); - if (enum_value == null) { - PrintWriter err = (PrintWriter) RT.ERR.deref(); - err.format("invalid enum value %s for enum type %s\n", name, enum_type.getFullName()); - } - return enum_value; - case MESSAGE: - PersistentProtocolBufferMap protobuf; - if (value instanceof PersistentProtocolBufferMap) { - protobuf = (PersistentProtocolBufferMap) value; - } else { - Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); - protobuf = PersistentProtocolBufferMap.construct(fieldDef, (IPersistentMap) value); - } - return protobuf.message(); - default: - return value; + case LONG: + if (value instanceof Long) + return value; + return new Long(((Integer)value).longValue()); + case INT: + if (value instanceof Integer) + return value; + return new Integer(((Long)value).intValue()); + case FLOAT: + if (value instanceof Integer) + return new Float((Integer)value * 1.0); + if (value instanceof Double) + return new Float((Double)value); + return value; + case DOUBLE: + if (value instanceof Integer) + return new Double((Integer)value * 1.0); + if (value instanceof Float) + return new Double((Float)value); + return value; + case ENUM: + String name = def.namingStrategy.protoName(value); + Descriptors.EnumDescriptor enum_type = field.getEnumType(); + Descriptors.EnumValueDescriptor enum_value = enum_type.findValueByName(name); + if (enum_value == null) { + PrintWriter err = (PrintWriter)RT.ERR.deref(); + err.format("invalid enum value %s for enum type %s\n", name, enum_type.getFullName()); + } + return enum_value; + case MESSAGE: + PersistentProtocolBufferMap protobuf; + if (value instanceof PersistentProtocolBufferMap) { + protobuf = (PersistentProtocolBufferMap)value; + } else { + Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), + this.def.namingStrategy); + protobuf = PersistentProtocolBufferMap.construct(fieldDef, (IPersistentMap)value); + } + return protobuf.message(); + default: + return value; } } - static protected GeneratedMessage.GeneratedExtension nullExtension(Descriptors.FieldDescriptor field) { + static protected GeneratedMessage.GeneratedExtension nullExtension( + Descriptors.FieldDescriptor field) { switch (field.getJavaType()) { - case LONG: return Extensions.nullLong; - case INT: return Extensions.nullInt; - case FLOAT: return Extensions.nullFloat; - case DOUBLE: return Extensions.nullDouble; - case STRING: return Extensions.nullString; - case ENUM: return Extensions.nullEnum; - default: return null; + case LONG: + return Extensions.nullLong; + case INT: + return Extensions.nullInt; + case FLOAT: + return Extensions.nullFloat; + case DOUBLE: + return Extensions.nullDouble; + case STRING: + return Extensions.nullString; + case ENUM: + return Extensions.nullEnum; + default: + return null; } } - protected void addRepeatedField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { + protected void addRepeatedField(DynamicMessage.Builder builder, + Descriptors.FieldDescriptor field, Object value) { try { builder.addRepeatedField(field, value); } catch (Exception e) { - String msg = String.format("error adding %s to %s field %s", - value, field.getJavaType().toString().toLowerCase(), - field.getFullName()); + String msg = String.format("error adding %s to %s field %s", value, + field.getJavaType().toString().toLowerCase(), field.getFullName()); throw new IllegalArgumentException(msg, e); } } - protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, Object value) { + protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescriptor field, + Object value) { try { builder.setField(field, value); } catch (IllegalArgumentException e) { String msg = String.format("error setting %s field %s to %s", - field.getJavaType().toString().toLowerCase(), - field.getFullName(), value); + field.getJavaType().toString().toLowerCase(), field.getFullName(), value); throw new IllegalArgumentException(msg, e); } } protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object key, Object value) { - if (key == null) return builder; + if (key == null) + return builder; Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) return builder; - if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) return builder; + if (field == null) + return builder; + if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) + return builder; boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { @@ -497,15 +549,15 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object Object map_field_by = def.mapFieldBy(field); if (map_field_by != null) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); - IPersistentMap map = (IPersistentMap) e.getValue(); + Map.Entry e = (Map.Entry)s.first(); + IPersistentMap map = (IPersistentMap)e.getValue(); Object k = e.getKey(); Object v = toProtoValue(field, map.assoc(map_field_by, k)); addRepeatedField(builder, field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { for (ISeq s = RT.seq(value); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry)s.first(); Object[] map = {k_key, e.getKey(), k_val, e.getValue()}; addRepeatedField(builder, field, toProtoValue(field, new PersistentArrayMap(map))); } @@ -514,10 +566,12 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object boolean isMap = (value instanceof IPersistentMap); for (ISeq s = RT.seq(value); s != null; s = s.next()) { if (isMap) { - Map.Entry e = (Map.Entry) s.first(); - k = e.getKey(); v = e.getValue(); + Map.Entry e = (Map.Entry)s.first(); + k = e.getKey(); + v = e.getValue(); } else { - k = s.first(); v = true; + k = s.first(); + v = true; } Object[] map = {k_item, k, k_exists, v}; addRepeatedField(builder, field, toProtoValue(field, new PersistentArrayMap(map))); @@ -529,7 +583,7 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object } else { Object v = toProtoValue(field, value); if (v instanceof DynamicMessage) { - v = ((DynamicMessage) builder.getField(field)).toBuilder().mergeFrom((DynamicMessage) v).build(); + v = ((DynamicMessage)builder.getField(field)).toBuilder().mergeFrom((DynamicMessage)v).build(); } setField(builder, field, v); } @@ -538,11 +592,12 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object } public PersistentProtocolBufferMap withMeta(IPersistentMap meta) { - if (meta == meta()) return this; + if (meta == meta()) + return this; return new PersistentProtocolBufferMap(meta, ext, def, message); } - public IPersistentMap meta(){ + public IPersistentMap meta() { return _meta; } @@ -594,31 +649,31 @@ public IPersistentMap assoc(Object key, Object value) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field != null) { - return new PersistentProtocolBufferMap(meta(), ext, def, - addField(builder(), field, value)); + return new PersistentProtocolBufferMap(meta(), ext, def, addField(builder(), field, value)); } else { - return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), - def, builder()); + return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), def, builder()); } } public IPersistentMap assocEx(Object key, Object value) throws Exception { - if(containsKey(key)) throw new Exception("Key already present"); + if (containsKey(key)) + throw new Exception("Key already present"); return assoc(key, value); } public IPersistentCollection cons(Object o) { DynamicMessage.Builder builder = builder(); if (o instanceof Map.Entry) { - Map.Entry e = (Map.Entry) o; + Map.Entry e = (Map.Entry)o; addField(builder, e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { - IPersistentVector v = (IPersistentVector) o; - if (v.count() != 2) throw new IllegalArgumentException("Vector arg to map conj must be a pair"); + IPersistentVector v = (IPersistentVector)o; + if (v.count() != 2) + throw new IllegalArgumentException("Vector arg to map conj must be a pair"); addField(builder, v.nth(0), v.nth(1)); } else { for (ISeq s = RT.seq(o); s != null; s = s.next()) { - Map.Entry e = (Map.Entry) s.first(); + Map.Entry e = (Map.Entry)s.first(); addField(builder, e.getKey(), e.getValue()); } } @@ -628,7 +683,7 @@ public IPersistentCollection cons(Object o) { public PersistentProtocolBufferMap append(IPersistentMap map) { PersistentProtocolBufferMap proto; if (map instanceof PersistentProtocolBufferMap) { - proto = (PersistentProtocolBufferMap) map; + proto = (PersistentProtocolBufferMap)map; } else { proto = construct(def, map); } @@ -644,7 +699,8 @@ public IPersistentMap without(Object key) throws Exception { } return new PersistentProtocolBufferMap(meta(), newExt, def, builder()); } - if (field.isRequired()) throw new Exception("Can't remove required field"); + if (field.isRequired()) + throw new Exception("Can't remove required field"); return new PersistentProtocolBufferMap(meta(), ext, def, builder().clearField(field)); } @@ -676,25 +732,28 @@ private static class Seq extends ASeq { private final MapEntry first; private final ISeq fields; - public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields){ + public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto, ISeq fields) { for (ISeq s = fields; s != null; s = s.next()) { - Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor) s.first(); + Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor)s.first(); Object k = proto.def.intern(field.getName()); - Object v = proto.valAt(k, sentinel); - if (v != sentinel) return new Seq(meta, proto, new MapEntry(k, v), s); + Object v = proto.valAt(k, sentinel); + if (v != sentinel) + return new Seq(meta, proto, new MapEntry(k, v), s); } return RT.seq(proto.ext); } - protected Seq(IPersistentMap meta, PersistentProtocolBufferMap proto, MapEntry first, ISeq fields){ + protected Seq(IPersistentMap meta, PersistentProtocolBufferMap proto, MapEntry first, + ISeq fields) { super(meta); - this.proto = proto; - this.first = first; + this.proto = proto; + this.first = first; this.fields = fields; } public Obj withMeta(IPersistentMap meta) { - if (meta != meta()) return new Seq(meta, proto, first, fields); + if (meta != meta()) + return new Seq(meta, proto, first, fields); return this; } @@ -706,4 +765,4 @@ public ISeq next() { return create(meta(), proto, fields.next()); } } -} \ No newline at end of file +} From 8cf48b0a0c0b3bd1f1a2115b86edb00c75d5006d Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 15:00:17 -0700 Subject: [PATCH 243/525] Fix code style with Eclipse. In particular, put {}s around our single-statement if/else clauses. --- .../core/PersistentProtocolBufferMap.java | 85 ++++++++++++++----- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index d61549d..bd514e6 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -55,27 +55,33 @@ public static final String nameStr(Object named) { } public static final NamingStrategy protobufNames = new NamingStrategy() { + @Override public String protoName(Object name) { return nameStr(name); } + @Override public Object clojureName(String name) { return Keyword.intern(name.toLowerCase()); } + @Override public String toString() { return "[protobuf names]"; } }; public static final NamingStrategy convertUnderscores = new NamingStrategy() { + @Override public String protoName(Object name) { return nameStr(name).replaceAll("-", "_"); } + @Override public Object clojureName(String name) { return Keyword.intern(name.replaceAll("_", "-").toLowerCase()); } + @Override public String toString() { return "[convert underscores]"; } @@ -103,8 +109,9 @@ public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { defCache = new ConcurrentHashMap(); ConcurrentHashMap previous = type_to_def.putIfAbsent(strat, defCache); - if (previous != null) + if (previous != null) { defCache = previous; + } } Def def = defCache.get(type); @@ -143,8 +150,9 @@ public DynamicMessage.Builder newBuilder() { } public Descriptors.FieldDescriptor fieldDescriptor(Object key) { - if (key == null) + if (key == null) { return null; + } if (key instanceof Descriptors.FieldDescriptor) { return (Descriptors.FieldDescriptor)key; @@ -182,15 +190,17 @@ public Object intern(String name) { if (nameCache == null) { nameCache = new ConcurrentHashMap(); ConcurrentHashMap existing = caches.putIfAbsent(namingStrategy, nameCache); - if (existing != null) + if (existing != null) { nameCache = existing; + } } Object clojureName = nameCache.get(name); if (clojureName == null) { clojureName = namingStrategy.clojureName(name); Object existing = nameCache.putIfAbsent(name, clojureName); - if (existing != null) + if (existing != null) { clojureName = existing; + } } return clojureName; } @@ -411,8 +421,9 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, DynamicMessage message = (DynamicMessage)value; // Total hack because getField() doesn't return an empty array for repeated messages. - if (field.isRepeated() && !message.isInitialized()) + if (field.isRepeated() && !message.isInitialized()) { return fromProtoValue(field, new ArrayList(), use_extensions); + } return new PersistentProtocolBufferMap(null, fieldDef, message); default: @@ -443,24 +454,30 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { switch (field.getJavaType()) { case LONG: - if (value instanceof Long) + if (value instanceof Long) { return value; + } return new Long(((Integer)value).longValue()); case INT: - if (value instanceof Integer) + if (value instanceof Integer) { return value; + } return new Integer(((Long)value).intValue()); case FLOAT: - if (value instanceof Integer) + if (value instanceof Integer) { return new Float((Integer)value * 1.0); - if (value instanceof Double) + } + if (value instanceof Double) { return new Float((Double)value); + } return value; case DOUBLE: - if (value instanceof Integer) + if (value instanceof Integer) { return new Double((Integer)value * 1.0); - if (value instanceof Float) + } + if (value instanceof Float) { return new Double((Float)value); + } return value; case ENUM: String name = def.namingStrategy.protoName(value); @@ -478,7 +495,7 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { } else { Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), this.def.namingStrategy); - protobuf = PersistentProtocolBufferMap.construct(fieldDef, (IPersistentMap)value); + protobuf = PersistentProtocolBufferMap.construct(fieldDef, value); } return protobuf.message(); default: @@ -529,13 +546,16 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip } protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object key, Object value) { - if (key == null) + if (key == null) { return builder; + } Descriptors.FieldDescriptor field = def.fieldDescriptor(key); - if (field == null) + if (field == null) { return builder; - if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) + } + if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) { return builder; + } boolean set = field.getOptions().getExtension(Extensions.set); if (field.isRepeated()) { @@ -591,16 +611,20 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object return builder; } + @Override public PersistentProtocolBufferMap withMeta(IPersistentMap meta) { - if (meta == meta()) + if (meta == meta()) { return this; + } return new PersistentProtocolBufferMap(meta, ext, def, message); } + @Override public IPersistentMap meta() { return _meta; } + @Override public boolean containsKey(Object key) { return protoContainsKey(key) || RT.booleanCast(RT.contains(ext, key)); } @@ -618,15 +642,18 @@ private boolean protoContainsKey(Object key) { private static final Object sentinel = new Object(); + @Override public IMapEntry entryAt(Object key) { Object value = valAt(key, sentinel); return (value == sentinel) ? null : new MapEntry(key, value); } + @Override public Object valAt(Object key) { return getValAt(key, true); } + @Override public Object valAt(Object key, Object notFound) { return getValAt(key, notFound, true); } @@ -645,6 +672,7 @@ public Object getValAt(Object key, Object notFound, boolean use_extensions) { } } + @Override public IPersistentMap assoc(Object key, Object value) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); @@ -655,12 +683,15 @@ public IPersistentMap assoc(Object key, Object value) { } } + @Override public IPersistentMap assocEx(Object key, Object value) throws Exception { - if (containsKey(key)) + if (containsKey(key)) { throw new Exception("Key already present"); + } return assoc(key, value); } + @Override public IPersistentCollection cons(Object o) { DynamicMessage.Builder builder = builder(); if (o instanceof Map.Entry) { @@ -668,8 +699,9 @@ public IPersistentCollection cons(Object o) { addField(builder, e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector)o; - if (v.count() != 2) + if (v.count() != 2) { throw new IllegalArgumentException("Vector arg to map conj must be a pair"); + } addField(builder, v.nth(0), v.nth(1)); } else { for (ISeq s = RT.seq(o); s != null; s = s.next()) { @@ -690,6 +722,7 @@ public PersistentProtocolBufferMap append(IPersistentMap map) { return new PersistentProtocolBufferMap(meta(), ext, def, builder().mergeFrom(proto.message())); } + @Override public IPersistentMap without(Object key) throws Exception { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) { @@ -699,16 +732,19 @@ public IPersistentMap without(Object key) throws Exception { } return new PersistentProtocolBufferMap(meta(), newExt, def, builder()); } - if (field.isRequired()) + if (field.isRequired()) { throw new Exception("Can't remove required field"); + } return new PersistentProtocolBufferMap(meta(), ext, def, builder().clearField(field)); } + @Override public Iterator iterator() { return new SeqIterator(seq()); } + @Override public int count() { int count = RT.count(ext); for (Descriptors.FieldDescriptor field : def.type.getFields()) { @@ -719,10 +755,12 @@ public int count() { return count; } + @Override public ISeq seq() { return Seq.create(null, this, RT.seq(def.type.getFields())); } + @Override public IPersistentCollection empty() { return new PersistentProtocolBufferMap(meta(), null, def, builder().clear()); } @@ -737,8 +775,9 @@ public static ISeq create(IPersistentMap meta, PersistentProtocolBufferMap proto Descriptors.FieldDescriptor field = (Descriptors.FieldDescriptor)s.first(); Object k = proto.def.intern(field.getName()); Object v = proto.valAt(k, sentinel); - if (v != sentinel) + if (v != sentinel) { return new Seq(meta, proto, new MapEntry(k, v), s); + } } return RT.seq(proto.ext); } @@ -751,16 +790,20 @@ protected Seq(IPersistentMap meta, PersistentProtocolBufferMap proto, MapEntry f this.fields = fields; } + @Override public Obj withMeta(IPersistentMap meta) { - if (meta != meta()) + if (meta != meta()) { return new Seq(meta, proto, first, fields); + } return this; } + @Override public Object first() { return first; } + @Override public ISeq next() { return create(meta(), proto, fields.next()); } From 3c99fd305e412c276599d1450e2c02edef57f75e Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 20 Apr 2012 15:04:20 -0700 Subject: [PATCH 244/525] Remove unqualified imports like clojure.lang.* --- .../core/PersistentProtocolBufferMap.java | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index bd514e6..232b7a5 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -10,23 +10,50 @@ package protobuf.core; -import clojure.lang.*; -import ordered_set.core.OrderedSet; -import java.util.*; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; -import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; import java.lang.reflect.InvocationTargetException; -import com.google.protobuf.DescriptorProtos.FieldOptions; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DescriptorProtos; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import ordered_set.core.OrderedSet; +import clojure.lang.APersistentMap; +import clojure.lang.ASeq; +import clojure.lang.IFn; +import clojure.lang.IMapEntry; +import clojure.lang.IObj; +import clojure.lang.IPersistentCollection; +import clojure.lang.IPersistentMap; +import clojure.lang.IPersistentVector; +import clojure.lang.ISeq; +import clojure.lang.ITransientMap; +import clojure.lang.ITransientSet; +import clojure.lang.Keyword; +import clojure.lang.MapEntry; +import clojure.lang.Numbers; +import clojure.lang.Obj; +import clojure.lang.PersistentArrayMap; +import clojure.lang.PersistentHashMap; +import clojure.lang.PersistentVector; +import clojure.lang.RT; +import clojure.lang.SeqIterator; +import clojure.lang.Sequential; +import clojure.lang.Symbol; +import clojure.lang.Var; + import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.DescriptorProtos.FieldOptions; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; import com.google.protobuf.GeneratedMessage; +import com.google.protobuf.InvalidProtocolBufferException; public class PersistentProtocolBufferMap extends APersistentMap implements IObj { From 85689c623dff9f8b3ae941a9d126be67dc6a1862 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 23 Apr 2012 13:46:57 -0700 Subject: [PATCH 245/525] Add Eclipse cruft to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2b9dd56..52e202e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ proto/google/protobuf/descriptor.proto docs protobuf-* protosrc +.classpath +.project +.settings From cb5230356498dfd1803531e9c3269fe51b7dab0f Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 23 Apr 2012 13:47:20 -0700 Subject: [PATCH 246/525] Use OrderedMap for protobuf maps --- project.clj | 2 +- src/protobuf/core/PersistentProtocolBufferMap.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index b24fd47..cb4185a 100644 --- a/project.clj +++ b/project.clj @@ -1,7 +1,7 @@ (defproject protobuf "0.6.0-beta14" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[clojure "1.2.0"] - [ordered-set "0.2.3"] + [ordered-set "0.3.0"] [useful "0.8.0-alpha1"] [fs "1.0.0"] [conch "0.2.0"] diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 232b7a5..57258aa 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -21,6 +21,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import ordered_map.core.OrderedMap; import ordered_set.core.OrderedSet; import clojure.lang.APersistentMap; import clojure.lang.ASeq; @@ -38,7 +39,6 @@ import clojure.lang.Numbers; import clojure.lang.Obj; import clojure.lang.PersistentArrayMap; -import clojure.lang.PersistentHashMap; import clojure.lang.PersistentVector; import clojure.lang.RT; import clojure.lang.SeqIterator; @@ -364,7 +364,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, Object map_field_by = def.mapFieldBy(field); DescriptorProtos.FieldOptions options = field.getOptions(); if (map_field_by != null) { - ITransientMap map = PersistentHashMap.EMPTY.asTransient(); + ITransientMap map = (ITransientMap)OrderedMap.EMPTY.asTransient(); while (iterator.hasNext()) { PersistentProtocolBufferMap v = (PersistentProtocolBufferMap)fromProtoValue(field, iterator.next()); @@ -390,7 +390,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, Descriptors.FieldDescriptor key_field = type.findFieldByName("key"); Descriptors.FieldDescriptor val_field = type.findFieldByName("val"); - ITransientMap map = PersistentHashMap.EMPTY.asTransient(); + ITransientMap map = (ITransientMap)OrderedMap.EMPTY.asTransient(); while (iterator.hasNext()) { DynamicMessage message = (DynamicMessage)iterator.next(); Object k = fromProtoValue(key_field, message.getField(key_field)); From 49f959e81d78e59e156721f7108c72bbb29adacf Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 23 Apr 2012 14:30:42 -0700 Subject: [PATCH 247/525] In map_field_by, assoc both the clojure name and the protobuf name. Should only matter in cases that are basically errors. --- src/protobuf/core/PersistentProtocolBufferMap.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 57258aa..06e5bbd 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -595,11 +595,12 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object } else { Object map_field_by = def.mapFieldBy(field); if (map_field_by != null) { + String field_name = def.namingStrategy.protoName(map_field_by); for (ISeq s = RT.seq(value); s != null; s = s.next()) { Map.Entry e = (Map.Entry)s.first(); IPersistentMap map = (IPersistentMap)e.getValue(); Object k = e.getKey(); - Object v = toProtoValue(field, map.assoc(map_field_by, k)); + Object v = toProtoValue(field, map.assoc(map_field_by, k).assoc(field_name, k)); addRepeatedField(builder, field, v); } } else if (field.getOptions().getExtension(Extensions.map)) { From 8ce79568d4d7dfcbd59ba99b3f3ffa0ad9a2e24b Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 23 Apr 2012 18:46:23 -0700 Subject: [PATCH 248/525] Lein2-friendly project.clj --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index cb4185a..6526cf5 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (defproject protobuf "0.6.0-beta14" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[clojure "1.2.0"] + :dependencies [[org.clojure/clojure "1.2.0"] [ordered-set "0.3.0"] [useful "0.8.0-alpha1"] [fs "1.0.0"] From 9a7c9162fd708c54b56dae68e3693ccb44755d6c Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 23 Apr 2012 18:46:34 -0700 Subject: [PATCH 249/525] Add test for our new use of OrderedMap --- test/protobuf/core_test.clj | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 0f4fa5f..f1808d4 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -1,5 +1,6 @@ (ns protobuf.core-test - (:use protobuf.core clojure.test) + (:use protobuf.core clojure.test + ordered-map.core) (:import (java.io PipedInputStream PipedOutputStream))) (def Foo (protodef protobuf.test.Core$Foo)) @@ -51,6 +52,16 @@ (is (= [1.2 3.4] (:doubles p))) (is (= [(float 0.01) (float 0.02)] (:floats p)))))) +(deftest test-ordered-append + (let [inputs (apply ordered-map (for [x (range 26) + entry [(str x) (str (char (+ (int \a) x)))]] + entry))] + (= (seq inputs) + (seq (reduce (fn [m [k v]] + (append m {:attr_map {k v}})) + (protobuf Foo) + inputs))))) + (deftest test-assoc (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :foo-by-id {1 {:label "one"} 2 {:label "two"}})] (let [p (assoc p :label "baz" :tags ["nuprin"])] From 56f5aef7e57e1ba75e8f0a445346a9d540298769 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 25 Apr 2012 13:08:17 -0700 Subject: [PATCH 250/525] Clarify nil-test --- src/protobuf/core.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index e47d2ea..8e5ed40 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -24,8 +24,7 @@ def (protodef def PersistentProtocolBufferMap$Def/convertUnderscores))) ([def ^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy] - (if (nil? def) - def + (when def (let [^Descriptors$Descriptor descriptor (if (instance? Descriptors$Descriptor def) def From 9036fc0bff867cad6fbbdbaa5bef0bce5921258d Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 25 Apr 2012 13:08:39 -0700 Subject: [PATCH 251/525] Participate in adjoin protocol instead of inventing append function --- src/protobuf/core.clj | 11 ++++++----- test/protobuf/core_test.clj | 29 +++++++++++++++-------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 8e5ed40..91591dd 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -2,6 +2,7 @@ (:use [protobuf.schema :only [field-schema]] [useful.fn :only [fix]] [clojure.java.io :only [input-stream output-stream]]) + (:require useful.utils) (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def PersistentProtocolBufferMap$Def$NamingStrategy Extensions) (com.google.protobuf GeneratedMessage CodedInputStream Descriptors$Descriptor) (java.io InputStream OutputStream) @@ -89,12 +90,12 @@ (.writeDelimitedTo p out)) (.flush out)))) -;; TODO make these functions nil-safe -(defn append - "Merge the given map into the protobuf. Equivalent to appending the byte representations." - [^PersistentProtocolBufferMap p map] - (.append p map)) +(extend-protocol useful.utils/Adjoin + PersistentProtocolBufferMap + (adjoin [^PersistentProtocolBufferMap this other] + (.append this other))) +;; TODO make this nil-safe? Or just delete it? (defn get-raw "Get value at key ignoring extension fields." [^PersistentProtocolBufferMap p key] diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index f1808d4..57ab99c 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -1,5 +1,6 @@ (ns protobuf.core-test (:use protobuf.core clojure.test + [useful.utils :only [adjoin]] ordered-map.core) (:import (java.io PipedInputStream PipedOutputStream))) @@ -34,31 +35,31 @@ (is (= ["little" "yellow" "different"] (:tags p))) (is (= "very" (:label p)))))) -(deftest test-append +(deftest test-adjoin (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2] :floats [0.01])] - (let [p (append p {:label "bar"})] + (let [p (adjoin p {:label "bar"})] (is (= 5 (:id p))) (is (= "bar" (:label p))) (is (= false (:deleted p))) (is (= ["little" "yellow"] (:tags p)))) - (let [p (append (assoc p :deleted true) p)] + (let [p (adjoin (assoc p :deleted true) p)] (is (= true (:deleted p)))) - (let [p (append p {:tags ["different"]})] + (let [p (adjoin p {:tags ["different"]})] (is (= ["little" "yellow" "different"] (:tags p)))) - (let [p (append p {:tags ["different"] :label "very"})] + (let [p (adjoin p {:tags ["different"] :label "very"})] (is (= ["little" "yellow" "different"] (:tags p))) (is (= "very" (:label p)))) - (let [p (append p {:doubles [3.4] :floats [0.02]})] + (let [p (adjoin p {:doubles [3.4] :floats [0.02]})] (is (= [1.2 3.4] (:doubles p))) (is (= [(float 0.01) (float 0.02)] (:floats p)))))) -(deftest test-ordered-append +(deftest test-ordered-adjoin (let [inputs (apply ordered-map (for [x (range 26) entry [(str x) (str (char (+ (int \a) x)))]] entry))] (= (seq inputs) (seq (reduce (fn [m [k v]] - (append m {:attr_map {k v}})) + (adjoin m {:attr_map {k v}})) (protobuf Foo) inputs))))) @@ -181,11 +182,11 @@ (let [p (protobuf Foo :counts {"foo" {:i 5 :d 5.0}})] (is (= 5 (get-in p [:counts "foo" :i]))) (is (= 5.0 (get-in p [:counts "foo" :d]))) - (let [p (append p {:counts {"foo" {:i 2 :d -2.4} "bar" {:i 99}}})] + (let [p (adjoin p {:counts {"foo" {:i 2 :d -2.4} "bar" {:i 99}}})] (is (= 7 (get-in p [:counts "foo" :i]))) (is (= 2.6 (get-in p [:counts "foo" :d]))) (is (= 99 (get-in p [:counts "bar" :i]))) - (let [p (append p {:counts {"foo" {:i -8 :d 4.06} "bar" {:i -66}}})] + (let [p (adjoin p {:counts {"foo" {:i -8 :d 4.06} "bar" {:i -66}}})] (is (= -1 (get-in p [:counts "foo" :i]))) (is (= 6.66 (get-in p [:counts "foo" :d]))) (is (= 33 (get-in p [:counts "bar" :i]))) @@ -201,7 +202,7 @@ (is (= 1978 (get-in p [:time :year]))) (is (= 11 (get-in p [:time :month]))) (is (= 24 (get-in p [:time :day]))) - (let [p (append p {:time {:year 1974 :month 1}})] + (let [p (adjoin p {:time {:year 1974 :month 1}})] (is (= 1974 (get-in p [:time :year]))) (is (= 1 (get-in p [:time :month]))) (is (= nil (get-in p [:time :day]))) @@ -218,7 +219,7 @@ (is (= "foo" (get p :str))) (is (= :a (get p :enu))) (is (= keyset (set (keys p)))) - (let [p (append p {:int nil :long nil :flt nil :dbl nil :str nil :enu nil})] + (let [p (adjoin p {:int nil :long nil :flt nil :dbl nil :str nil :enu nil})] (is (= nil (get p :int))) (is (= nil (get p :long))) (is (= nil (get p :flt))) @@ -229,13 +230,13 @@ (testing "nullable successions" (let [p (protobuf Bar :label "foo")] (is (= "foo" (get p :label))) - (let [p (append p {:label nil})] + (let [p (adjoin p {:label nil})] (is (= nil (get p :label))) (is (= ["foo" ""] (get-raw p :label)))))) (testing "repeated nullable" (let [p (protobuf Bar :labels ["foo" "bar"])] (is (= ["foo" "bar"] (get p :labels))) - (let [p (append p {:labels [nil]})] + (let [p (adjoin p {:labels [nil]})] (is (= ["foo" "bar" nil] (get p :labels))) (is (= ["foo" "bar" ""] (get-raw p :labels)))))))) From a2ae17aaed3b8b3dadcaedd908c9002b9545f0df Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 25 Apr 2012 13:35:29 -0700 Subject: [PATCH 252/525] New version of useful contains Adjoin protocol --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 6526cf5..e754222 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.2.0"] [ordered-set "0.3.0"] - [useful "0.8.0-alpha1"] + [useful "0.8.0-alpha4"] [fs "1.0.0"] [conch "0.2.0"] [schematic "0.0.5"]] From 1e6b2e1b861e843a3b8d27db81e9cf7928184f0c Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 25 Apr 2012 13:35:42 -0700 Subject: [PATCH 253/525] checksum-deps in project.clj for convenience --- project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/project.clj b/project.clj index e754222..f46b7c0 100644 --- a/project.clj +++ b/project.clj @@ -14,4 +14,5 @@ ;; jar to implicitly clean no matter what, wiping stuff. ;; This prevents that. :disable-implicit-clean true + :checksum-deps true :java-source-path "src") From ed9e273e9c3d2cf21a7b885d64c7fd0db74249a2 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 25 Apr 2012 13:48:45 -0700 Subject: [PATCH 254/525] beta 15 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index f46b7c0..b3b8808 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta14" +(defproject protobuf "0.6.0-beta15" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.2.0"] [ordered-set "0.3.0"] From 299f0c51fef4c442c72772cb00e46bd3656424ce Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Thu, 26 Apr 2012 17:29:01 -0700 Subject: [PATCH 255/525] 0.6.0-beta16: update to ordered-collections 0.4.0 and useful 0.8.0-alpha5 --- project.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index b3b8808..da94fbf 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject protobuf "0.6.0-beta15" +(defproject protobuf "0.6.0-beta16" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.2.0"] - [ordered-set "0.3.0"] - [useful "0.8.0-alpha4"] + [ordered-collections "0.4.0"] + [useful "0.8.0-alpha5"] [fs "1.0.0"] [conch "0.2.0"] [schematic "0.0.5"]] From 5e2b82321e233c13869e5da13e31fcbb557056d8 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 1 May 2012 16:48:43 -0700 Subject: [PATCH 256/525] Stop putting schema data on the codec as metadata --- src/protobuf/codec.clj | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/protobuf/codec.clj b/src/protobuf/codec.clj index 35419c4..0d26d6b 100644 --- a/src/protobuf/codec.clj +++ b/src/protobuf/codec.clj @@ -48,6 +48,8 @@ (protobuf proto val)))))) (fix repeated #(gloss/repeated (gloss/finite-frame (length-prefix proto) %) - :prefix :none)) - (with-meta {:schema (schema/dissoc-fields (protobuf-schema proto) - len-key reset-key)})))) + :prefix :none))))) + +(defn codec-schema [proto] + (schema/dissoc-fields (protobuf-schema proto) + len-key reset-key)) From 724c161e4a0b0bcacc921b450a087d9f571e6ffd Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 4 May 2012 16:00:52 -0700 Subject: [PATCH 257/525] Move catbytes to flatland/io --- project.clj | 2 +- test/protobuf/core_test.clj | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/project.clj b/project.clj index da94fbf..a09a7da 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [conch "0.2.0"] [schematic "0.0.5"]] :dev-dependencies [[gloss "0.2.0-rc1"] - [io "0.1.0-alpha2"]] + [io "0.2.0-beta2"]] :hooks [leiningen.protobuf] :eval-in-leiningen true ;; Bug in the current 1.x branch of Leiningen causes diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 57ab99c..9d132e4 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -1,5 +1,6 @@ (ns protobuf.core-test (:use protobuf.core clojure.test + [io.core :only [catbytes]] [useful.utils :only [adjoin]] ordered-map.core) (:import (java.io PipedInputStream PipedOutputStream))) @@ -11,16 +12,6 @@ (def Response (protodef protobuf.test.Core$Response)) (def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) -(defn catbytes [& args] - (let [out-buf (byte-array (reduce + (map count args)))] - (loop [offset 0, args args] - (if-let [[^bytes array & more] (seq args)] - (let [size (count array)] - (System/arraycopy array 0 - out-buf offset size) - (recur (+ size offset) more)) - out-buf)))) - (deftest test-conj (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] (let [p (conj p {:label "bar"})] From 4d8e781acaa038956d4b79bdfc674ac64f7ebcd3 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 1 May 2012 16:49:04 -0700 Subject: [PATCH 258/525] beta17, lol? --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index a09a7da..e5cd574 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta16" +(defproject protobuf "0.6.0-beta17" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.2.0"] [ordered-collections "0.4.0"] From 6d5e70a3fe70ba26f3c8ef1e31484752206ca5c5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 11 May 2012 18:22:28 -0700 Subject: [PATCH 259/525] add tests for adjoining protobuf extensions --- test/protobuf/core_test.clj | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index 9d132e4..dd73e6e 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -42,7 +42,24 @@ (is (= "very" (:label p)))) (let [p (adjoin p {:doubles [3.4] :floats [0.02]})] (is (= [1.2 3.4] (:doubles p))) - (is (= [(float 0.01) (float 0.02)] (:floats p)))))) + (is (= [(float 0.01) (float 0.02)] (:floats p))))) + (testing "adjoining works with set extension" + (let [p (protobuf Foo :tag-set #{"foo" "bar" "baz"}) + q (protobuf Foo :tag-set {"bar" false "foo" false "bap" true}) + r (adjoin p q)] + (is (= #{"foo" "bar" "baz"} (p :tag-set))) + (is (= #{"bap"} (q :tag-set))) + (is (= #{"bap" "baz"} (r :tag-set))))) + (testing "adjoining works with counters" + (let [p (protobuf Foo :counts {"foo" {:i 5 :d 5.0}}) + q (protobuf Foo :counts {"foo" {:i 8 :d -3.0}}) + r (adjoin p q)] + (is (= 5 (get-in p [:counts "foo" :i]))) + (is (= 5.0 (get-in p [:counts "foo" :d]))) + (is (= 8 (get-in q [:counts "foo" :i]))) + (is (= -3.0 (get-in q [:counts "foo" :d]))) + (is (= 13 (get-in r [:counts "foo" :i]))) + (is (= 2.0 (get-in r [:counts "foo" :d])))))) (deftest test-ordered-adjoin (let [inputs (apply ordered-map (for [x (range 26) From 32cf31bf7b58e17a53c1c46448488eefbd859c1f Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 16 May 2012 12:18:01 -0700 Subject: [PATCH 260/525] Update to new useful --- project.clj | 4 ++-- src/protobuf/core.clj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index e5cd574..d9918a4 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject protobuf "0.6.0-beta17" +(defproject protobuf "0.6.0-beta18" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :dependencies [[org.clojure/clojure "1.2.0"] [ordered-collections "0.4.0"] - [useful "0.8.0-alpha5"] + [useful "0.8.2-alpha1"] [fs "1.0.0"] [conch "0.2.0"] [schematic "0.0.5"]] diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 91591dd..206f610 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -92,7 +92,7 @@ (extend-protocol useful.utils/Adjoin PersistentProtocolBufferMap - (adjoin [^PersistentProtocolBufferMap this other] + (adjoin-onto [^PersistentProtocolBufferMap this other] (.append this other))) ;; TODO make this nil-safe? Or just delete it? From 96dbb5436cd405d2bc4bd0c7428c6ee8e6e64650 Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 19 Jun 2012 14:35:05 -0700 Subject: [PATCH 261/525] upgraded to be compatible with lein2 --- .gitignore | 1 + src/leiningen/protobuf.clj | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 52e202e..9468b9f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ protosrc .classpath .project .settings +target/ diff --git a/src/leiningen/protobuf.clj b/src/leiningen/protobuf.clj index 034dce3..813035d 100644 --- a/src/leiningen/protobuf.clj +++ b/src/leiningen/protobuf.clj @@ -3,7 +3,7 @@ (:use [clojure.string :only [join]] [leiningen.help :only [help-for]] [leiningen.javac :only [javac]] - [leiningen.util.paths :only [get-os]] + [leiningen.core.eval :only [get-os]] [robert.hooke :only [add-hook]]) (:require [clojure.java.io :as io] [fs.core :as fs] @@ -144,7 +144,7 @@ (if (not (= (sh/exit-code protoc-result) 0)) (println "ERROR: " (sh/stream-to-string protoc-result :err)))))) (binding [*compile?* false] - (javac (assoc project :java-source-path dest-path))))))) + (javac (assoc project :java-source-paths [dest-path]))))))) (defn compile-google-protobuf "Compile com.google.protobuf.*" From b1c730a055f5d0bce6c3291c907513bc5f2ddf2b Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 19 Jun 2012 15:17:44 -0700 Subject: [PATCH 262/525] upgraded to clojure 1.3.0 and changed Exceptions to RuntimeExceptions in assocEx and without methods --- project.clj | 4 ++-- src/protobuf/core/PersistentProtocolBufferMap.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index d9918a4..68955cd 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ (defproject protobuf "0.6.0-beta18" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[org.clojure/clojure "1.2.0"] + :dependencies [[org.clojure/clojure "1.3.0"] [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [fs "1.0.0"] @@ -15,4 +15,4 @@ ;; This prevents that. :disable-implicit-clean true :checksum-deps true - :java-source-path "src") + :java-source-paths ["src"]) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 06e5bbd..08980bf 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -712,9 +712,9 @@ public IPersistentMap assoc(Object key, Object value) { } @Override - public IPersistentMap assocEx(Object key, Object value) throws Exception { + public IPersistentMap assocEx(Object key, Object value) { if (containsKey(key)) { - throw new Exception("Key already present"); + throw new RuntimeException("Key already present"); } return assoc(key, value); } @@ -751,7 +751,7 @@ public PersistentProtocolBufferMap append(IPersistentMap map) { } @Override - public IPersistentMap without(Object key) throws Exception { + public IPersistentMap without(Object key) { Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) { IPersistentMap newExt = (IPersistentMap)RT.dissoc(ext, key); @@ -761,7 +761,7 @@ public IPersistentMap without(Object key) throws Exception { return new PersistentProtocolBufferMap(meta(), newExt, def, builder()); } if (field.isRequired()) { - throw new Exception("Can't remove required field"); + throw new RuntimeException("Can't remove required field"); } return new PersistentProtocolBufferMap(meta(), ext, def, builder().clearField(field)); From 1880cb72f1e2463c96193981907630ed9f81f1c6 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 29 Jun 2012 18:26:16 -0500 Subject: [PATCH 263/525] Move leiningen plugin to a subproject. --- lein-protobuf/project.clj | 10 ++++++++++ {src => lein-protobuf/src}/leiningen/protobuf.clj | 0 project.clj | 12 +++++------- 3 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 lein-protobuf/project.clj rename {src => lein-protobuf/src}/leiningen/protobuf.clj (100%) diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj new file mode 100644 index 0000000..23e853e --- /dev/null +++ b/lein-protobuf/project.clj @@ -0,0 +1,10 @@ +(defproject lein-protobuf "0.1.0-alpha1" + :description "Leiningen plugin for clojure-protobuf." + :dependencies [[fs "1.0.0"] + [conch "0.2.0"]] + :eval-in-leiningen true + ;; Bug in the current 1.x branch of Leiningen causes + ;; jar to implicitly clean no matter what, wiping stuff. + ;; This prevents that. + :disable-implicit-clean true + :checksum-deps true) diff --git a/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj similarity index 100% rename from src/leiningen/protobuf.clj rename to lein-protobuf/src/leiningen/protobuf.clj diff --git a/project.clj b/project.clj index 68955cd..d8af9c4 100644 --- a/project.clj +++ b/project.clj @@ -1,15 +1,13 @@ -(defproject protobuf "0.6.0-beta18" +(defproject protobuf "0.6.0-beta19" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." - :dependencies [[org.clojure/clojure "1.3.0"] + :dependencies [[org.clojure/clojure "1.4.0"] [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] - [fs "1.0.0"] - [conch "0.2.0"] [schematic "0.0.5"]] - :dev-dependencies [[gloss "0.2.0-rc1"] - [io "0.2.0-beta2"]] + :plugins [[lein-protobuf "0.1.0-alpha1"]] + :profiles {:dev {:dependencies [[gloss "0.2.1"] + [io "0.2.0-beta2"]]}} :hooks [leiningen.protobuf] - :eval-in-leiningen true ;; Bug in the current 1.x branch of Leiningen causes ;; jar to implicitly clean no matter what, wiping stuff. ;; This prevents that. From 80dd186fa7b260cef5581802bd124bbe06505ad5 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Fri, 29 Jun 2012 16:40:18 -0700 Subject: [PATCH 264/525] Use Number interface instead of wonky special-cases. This code broke on 1.4 because we were assuming nobody would ever use a value for a FLOAT field, apparently. --- .../core/PersistentProtocolBufferMap.java | 26 +++---------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 08980bf..bc6aa94 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -481,31 +481,13 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { switch (field.getJavaType()) { case LONG: - if (value instanceof Long) { - return value; - } - return new Long(((Integer)value).longValue()); + return ((Number)value).longValue(); case INT: - if (value instanceof Integer) { - return value; - } - return new Integer(((Long)value).intValue()); + return ((Number)value).intValue(); case FLOAT: - if (value instanceof Integer) { - return new Float((Integer)value * 1.0); - } - if (value instanceof Double) { - return new Float((Double)value); - } - return value; + return ((Number)value).floatValue(); case DOUBLE: - if (value instanceof Integer) { - return new Double((Integer)value * 1.0); - } - if (value instanceof Float) { - return new Double((Float)value); - } - return value; + return ((Number)value).doubleValue(); case ENUM: String name = def.namingStrategy.protoName(value); Descriptors.EnumDescriptor enum_type = field.getEnumType(); From 9cbc07bf8ffd4c1d14012f480c7afbd15a9f4872 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 29 Jun 2012 18:44:53 -0500 Subject: [PATCH 265/525] Remove old comment and code about target and lein 1. --- lein-protobuf/src/leiningen/protobuf.clj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index 813035d..5b9d850 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -20,12 +20,8 @@ (java.net.URL. (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) -;; In lein 1 there was a :target-dir key that defaulted to "." and wasn't really -;; used much. In lein 2, things that are generated by plugins and such are -;; supposed to go in :target-path. This will default to 'target'. We don't want -;; to wait for this, so we'll simulate it here. (defn target [project] - (doto (io/file (or (:target-path project) "target")) + (doto (io/file (:target-path project)) .mkdirs)) (defn- proto-dependencies From 600e87e5ec72b1d2d1a0e13585dff2b68b98356d Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:31:28 -0500 Subject: [PATCH 266/525] Don't copy descriptor if it is already there. This is necessary to prevent the modtime of proto from always being greater than the compilation directory. --- lein-protobuf/src/leiningen/protobuf.clj | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index 5b9d850..a778c2a 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -48,7 +48,7 @@ (recur (into files (proto-dependencies proto-file)))))))))) (defn modtime [dir] - (let [files (-> dir io/file file-seq rest)] + (let [files (->> dir io/file file-seq rest)] (if (empty? files) 0 (apply max (map fs/mod-time files))))) @@ -127,7 +127,7 @@ dest-path (.getPath dest) proto-path (.getAbsoluteFile (io/file (or (:proto-path project) "proto")))] (when (or (> (modtime proto-path) (modtime dest)) - (> (modtime proto-path) (modtime "classes"))) + (> (modtime proto-path) (modtime (str target "/classes")))) (.mkdirs dest) (.mkdirs proto-path) (doseq [proto protos] @@ -145,14 +145,17 @@ (defn compile-google-protobuf "Compile com.google.protobuf.*" [project] - (let [proto-files (io/file "proto/google/protobuf") - target (target project)] - (.mkdirs proto-files) - (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") - (io/file proto-files "descriptor.proto")) - (protoc project - ["google/protobuf/descriptor.proto"] - (io/file target srcdir "java/src/main/java")))) + (let [proto-files (io/file (get project :proto-path "proto") "google/protobuf") + target (target project) + descriptor (io/file proto-files "descriptor.proto")] + (when-not (.exists descriptor) + (fetch project) + (.mkdirs proto-files) + (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") + descriptor) + (protoc project + ["google/protobuf/descriptor.proto"] + (io/file target srcdir "java/src/main/java"))))) (defn compile "Compile protocol buffer files located in proto dir." @@ -160,8 +163,7 @@ (apply compile project (proto-files (io/file (or (:proto-path project) "proto"))))) ([project & files] (install project) - (when (= "protobuf" (:name project)) - (fetch project) + (when (and (= "protobuf" (:name project))) (compile-google-protobuf project)) (protoc project files))) From 62474f7a7a2bed44e52e00b76ec8bad65f7aeb4b Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:47:41 -0500 Subject: [PATCH 267/525] Move symlink to proto to lein-protobuf (so it ends up in that jar). --- lein-protobuf/resources/proto | 1 + resources/proto | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 lein-protobuf/resources/proto delete mode 120000 resources/proto diff --git a/lein-protobuf/resources/proto b/lein-protobuf/resources/proto new file mode 120000 index 0000000..b943f93 --- /dev/null +++ b/lein-protobuf/resources/proto @@ -0,0 +1 @@ +../../proto \ No newline at end of file diff --git a/resources/proto b/resources/proto deleted file mode 120000 index 4d1e2d8..0000000 --- a/resources/proto +++ /dev/null @@ -1 +0,0 @@ -../proto/ \ No newline at end of file From 16aff061041aa22f263378c09430041ba0eca8d0 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:48:35 -0500 Subject: [PATCH 268/525] lein-protobuf 0.1.0 --- lein-protobuf/project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj index 23e853e..4543766 100644 --- a/lein-protobuf/project.clj +++ b/lein-protobuf/project.clj @@ -1,4 +1,4 @@ -(defproject lein-protobuf "0.1.0-alpha1" +(defproject lein-protobuf "0.1.0" :description "Leiningen plugin for clojure-protobuf." :dependencies [[fs "1.0.0"] [conch "0.2.0"]] From 45bb0c9df5cb02498fe03faed727f9ab8345b1d7 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:48:52 -0500 Subject: [PATCH 269/525] Set real version of lein-protobuf. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d8af9c4..8567f72 100644 --- a/project.clj +++ b/project.clj @@ -4,7 +4,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.1.0-alpha1"]] + :plugins [[lein-protobuf "0.1.0"]] :profiles {:dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} :hooks [leiningen.protobuf] From a8b446f8f070adb52b3727f46fafc4aeacda5345 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:50:03 -0500 Subject: [PATCH 270/525] Add license info and URL. --- lein-protobuf/project.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj index 4543766..c561421 100644 --- a/lein-protobuf/project.clj +++ b/lein-protobuf/project.clj @@ -1,5 +1,8 @@ (defproject lein-protobuf "0.1.0" :description "Leiningen plugin for clojure-protobuf." + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :url "https://github.com/flatland/protobuf" :dependencies [[fs "1.0.0"] [conch "0.2.0"]] :eval-in-leiningen true From 88c15de81184b9cf16251735b310740f5ef44ca6 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 2 Jul 2012 17:56:29 -0500 Subject: [PATCH 271/525] Add :url and :license keys to the protobuf project. --- project.clj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project.clj b/project.clj index 8567f72..48be016 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,8 @@ (defproject protobuf "0.6.0-beta19" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :url "https://github.com/flatland/clojure-protobuf" :dependencies [[org.clojure/clojure "1.4.0"] [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] From a8c96e48e2bedf4c916bdbd3d1e031f81f41801c Mon Sep 17 00:00:00 2001 From: Anthony Marcar Date: Tue, 10 Jul 2012 10:13:45 -0700 Subject: [PATCH 272/525] upgraded fs to 1.2.0 --- lein-protobuf/project.clj | 2 +- lein-protobuf/src/leiningen/protobuf.clj | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj index c561421..c863c66 100644 --- a/lein-protobuf/project.clj +++ b/lein-protobuf/project.clj @@ -3,7 +3,7 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/flatland/protobuf" - :dependencies [[fs "1.0.0"] + :dependencies [[fs "1.2.0"] [conch "0.2.0"]] :eval-in-leiningen true ;; Bug in the current 1.x branch of Leiningen causes diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index a778c2a..31c3299 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -7,6 +7,7 @@ [robert.hooke :only [add-hook]]) (:require [clojure.java.io :as io] [fs.core :as fs] + [fs.compression :as fs-zip] [conch.core :as sh]) (:import java.util.zip.ZipFile)) @@ -81,7 +82,7 @@ (with-open [stream (.openStream url)] (io/copy stream (io/file zipped))) (println "Unzipping" zipfile "to" target) - (fs/unzip zipped target))))) + (fs-zip/unzip zipped target))))) (defn uninstall "Remove protoc if it is installed." From c51801832bfa1635dd07526273e7ea57cc915ae1 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Thu, 12 Jul 2012 18:08:19 -0500 Subject: [PATCH 273/525] Fix descriptor compilation in the protobuf project. Fixes #33. --- lein-protobuf/src/leiningen/protobuf.clj | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index 31c3299..efa91bd 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -148,15 +148,17 @@ [project] (let [proto-files (io/file (get project :proto-path "proto") "google/protobuf") target (target project) - descriptor (io/file proto-files "descriptor.proto")] - (when-not (.exists descriptor) + descriptor (io/file proto-files "descriptor.proto") + out (io/file target srcdir "java/src/main/java")] + (when-not (and (.exists descriptor) + (.exists (io/file out "com/google/protobuf/DescriptorProtos.java"))) (fetch project) (.mkdirs proto-files) (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") descriptor) (protoc project ["google/protobuf/descriptor.proto"] - (io/file target srcdir "java/src/main/java"))))) + out)))) (defn compile "Compile protocol buffer files located in proto dir." From 493642c8721c29717af16d86437e4bc6d6e30f32 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 13 Jul 2012 17:45:30 -0700 Subject: [PATCH 274/525] run protoc out of the protobuf source directory Why? So we don't have to try to sudo. Also, cache the protobuf source with build artifacts in ~/.m2 to prevent redownloading and reinstalling. --- lein-protobuf/project.clj | 2 +- lein-protobuf/src/leiningen/protobuf.clj | 96 +++++++++--------------- project.clj | 2 +- 3 files changed, 37 insertions(+), 63 deletions(-) diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj index c863c66..97e800f 100644 --- a/lein-protobuf/project.clj +++ b/lein-protobuf/project.clj @@ -1,4 +1,4 @@ -(defproject lein-protobuf "0.1.0" +(defproject lein-protobuf "0.1.1" :description "Leiningen plugin for clojure-protobuf." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index efa91bd..7011f68 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -12,14 +12,16 @@ (:import java.util.zip.ZipFile)) (def version "2.3.0") -(def srcdir (str "protobuf-" version)) +(def cache (format "%s/.m2/src/com/google/protobuf/%s" (System/getProperty "user.home") version)) (def zipfile (format "protobuf-%s.zip" version)) +(def srcdir (format "%s/protobuf-%s" cache version)) +(def protoc (format "%s/src/protoc" srcdir)) (def ^{:dynamic true} *compile?* true) (def url (java.net.URL. - (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" version))) + (format "http://protobuf.googlecode.com/files/%s" zipfile))) (defn target [project] (doto (io/file (:target-path project)) @@ -63,10 +65,6 @@ (for [file (rest (file-seq dir)) :when (proto-file? file)] (.substring (.getPath file) (inc (count (.getPath dir)))))) -(defn installed? [] - (try (.contains (sh/stream-to-string (sh/proc "protoc" "--version") :out) version) - (catch java.io.IOException e))) - (defn read-pass [] (print "Password: ") (flush) @@ -75,53 +73,35 @@ (defn fetch "Fetch protocol-buffer source and unzip it." [project] - (let [target (target project)] - (when-not (.exists (io/file target srcdir)) - (let [zipped (io/file target zipfile)] + (let [srcdir (io/file srcdir)] + (when-not (.exists srcdir) + (.mkdirs srcdir) + (let [zipped (io/file cache zipfile)] (println "Downloading" zipfile) (with-open [stream (.openStream url)] (io/copy stream (io/file zipped))) (println "Unzipping" zipfile "to" target) - (fs-zip/unzip zipped target))))) + (fs-zip/unzip zipped cache))))) -(defn uninstall - "Remove protoc if it is installed." - [project] - (when (installed?) - (let [password (read-pass) - proc (sh/proc "sudo" "-S" "make" "uninstall" - :dir (io/file (target project) srcdir))] - (sh/feed-from-string proc (str password "\n")) - (sh/stream-to-out proc :out)))) - -(defn install - "Compile and install protoc to /usr/local." +(defn build-protoc + "Compile protoc from source." [project] - (when-not (installed?) - (fetch project) - (let [source (io/file (target project) srcdir)] - (when-not (.exists (io/file source "src" "protoc")) - (fs/chmod "+x" (io/file source "configure")) - (fs/chmod "+x" (io/file source "install-sh")) + (let [srcdir (io/file srcdir) + protoc (io/file protoc)] + (when-not (.exists protoc) + (fetch project) + (when-not (.exists (io/file srcdir "src" "protoc")) + (fs/chmod "+x" (io/file srcdir "configure")) + (fs/chmod "+x" (io/file srcdir "install-sh")) (println "Configuring protoc") - (sh/stream-to-out (sh/proc "./configure" :dir source) :out) + (sh/stream-to-out (sh/proc "./configure" :dir srcdir) :out) (println "Running 'make'") - (sh/stream-to-out (sh/proc "make" :dir source) :out)) - (println "Installing") - (let [password (str (read-pass) "\n") - opts {:dir source :input-string (str password "\n")} - proc (if (= :linux (get-os)) - (sh/proc "script" "-q" "-c" "sudo -S make install" "/dev/null" - :dir source) - (sh/proc "sudo" "-S" "make" "install" - :dir source))] - (sh/feed-from-string proc password) - (sh/stream-to-out proc :out))))) - -(defn protoc + (sh/stream-to-out (sh/proc "make" :dir srcdir) :out))))) + +(defn compile-protobuf "Create .java and .class files from the provided .proto files." ([project protos] - (protoc project protos (io/file (target project) "protosrc"))) + (compile-protobuf project protos (io/file (target project) "protosrc"))) ([project protos dest] (let [target (target project) dest (.getAbsoluteFile dest) @@ -133,7 +113,7 @@ (.mkdirs proto-path) (doseq [proto protos] (extract-dependencies proto-path proto target) - (let [args ["protoc" proto (str "--java_out=" dest-path) "-I." + (let [args [protoc proto (str "--java_out=" dest-path) "-I." (str "-I" (.getAbsoluteFile (io/file target "proto"))) (str "-I" proto-path)]] (println " > " (join " " args)) @@ -149,26 +129,26 @@ (let [proto-files (io/file (get project :proto-path "proto") "google/protobuf") target (target project) descriptor (io/file proto-files "descriptor.proto") - out (io/file target srcdir "java/src/main/java")] + out (io/file srcdir "java/src/main/java")] (when-not (and (.exists descriptor) (.exists (io/file out "com/google/protobuf/DescriptorProtos.java"))) (fetch project) (.mkdirs proto-files) - (io/copy (io/file target srcdir "src/google/protobuf/descriptor.proto") + (io/copy (io/file srcdir "src/google/protobuf/descriptor.proto") descriptor) - (protoc project - ["google/protobuf/descriptor.proto"] - out)))) + (compile-protobuf project + ["google/protobuf/descriptor.proto"] + out)))) (defn compile "Compile protocol buffer files located in proto dir." ([project] (apply compile project (proto-files (io/file (or (:proto-path project) "proto"))))) ([project & files] - (install project) + (build-protoc project) (when (and (= "protobuf" (:name project))) (compile-google-protobuf project)) - (protoc project files))) + (compile-protobuf project files))) (add-hook #'javac (fn [f & args] @@ -176,13 +156,7 @@ (compile (first args))) (apply f args))) -(defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." - :help-arglists '([subtask & args]) - :subtasks [#'install #'uninstall #'compile]} - protobuf - ([project] (println (help-for "protobuf"))) - ([project subtask & args] - (case subtask - "install" (apply install project args) - "uninstall" (apply uninstall project args) - "compile" (apply compile project args)))) +(defn protobuf + "Task for compiling protobuf libraries." + [project & args] + (apply compile project args)) diff --git a/project.clj b/project.clj index 48be016..1b2f804 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.1.0"]] + :plugins [[lein-protobuf "0.1.1"]] :profiles {:dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} :hooks [leiningen.protobuf] From 0518876396a3d22725c3f7bd455a8d235ba04903 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 13:46:27 -0700 Subject: [PATCH 275/525] use ~/.lein/cache/lein-protobuf instead of hijacking ~/.m2 --- lein-protobuf/src/leiningen/protobuf.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index 7011f68..876fc23 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -4,6 +4,7 @@ [leiningen.help :only [help-for]] [leiningen.javac :only [javac]] [leiningen.core.eval :only [get-os]] + [leiningen.core.user :only [leiningen-home]] [robert.hooke :only [add-hook]]) (:require [clojure.java.io :as io] [fs.core :as fs] @@ -12,7 +13,7 @@ (:import java.util.zip.ZipFile)) (def version "2.3.0") -(def cache (format "%s/.m2/src/com/google/protobuf/%s" (System/getProperty "user.home") version)) +(def cache (str (leiningen-home) "/cache/lein-protobuf")) (def zipfile (format "protobuf-%s.zip" version)) (def srcdir (format "%s/protobuf-%s" cache version)) (def protoc (format "%s/src/protoc" srcdir)) From c84bb5a9214b89fbcd7f0f30f555387662e7f98d Mon Sep 17 00:00:00 2001 From: Travis Vachon Date: Mon, 9 Jul 2012 16:37:57 -0400 Subject: [PATCH 276/525] Allow protobufs version to be specified in the project file This lets us use this task with protobufs 2.4.1. Note that this does not guarantee that the top level project's Clojure library will be compatible with the generated code - change versions at your own risk. --- lein-protobuf/src/leiningen/protobuf.clj | 81 +++++++++++++++--------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index 876fc23..c12d268 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -12,17 +12,18 @@ [conch.core :as sh]) (:import java.util.zip.ZipFile)) -(def version "2.3.0") -(def cache (str (leiningen-home) "/cache/lein-protobuf")) -(def zipfile (format "protobuf-%s.zip" version)) -(def srcdir (format "%s/protobuf-%s" cache version)) -(def protoc (format "%s/src/protoc" srcdir)) +(def default-version "2.3.0") +(defn version + [project] + (or (:protobuf-version project) "default-version")) +(defn srcdir [project] (str "protobuf-" (version project))) +(defn zipfile [project] (format "protobuf-%s.zip" (version project))) (def ^{:dynamic true} *compile?* true) -(def url +(defn url [project] (java.net.URL. - (format "http://protobuf.googlecode.com/files/%s" zipfile))) + (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" (version project)))) (defn target [project] (doto (io/file (:target-path project)) @@ -66,6 +67,10 @@ (for [file (rest (file-seq dir)) :when (proto-file? file)] (.substring (.getPath file) (inc (count (.getPath dir)))))) +(defn installed? [project] + (try (.contains (sh/stream-to-string (sh/proc "protoc" "--version") :out) (version project)) + (catch java.io.IOException e))) + (defn read-pass [] (print "Password: ") (flush) @@ -74,26 +79,34 @@ (defn fetch "Fetch protocol-buffer source and unzip it." [project] - (let [srcdir (io/file srcdir)] - (when-not (.exists srcdir) - (.mkdirs srcdir) - (let [zipped (io/file cache zipfile)] - (println "Downloading" zipfile) - (with-open [stream (.openStream url)] + (let [target (target project)] + (when-not (.exists (io/file target (srcdir project))) + (let [zipped (io/file target (zipfile project))] + (println "Downloading" (zipfile project)) + (with-open [stream (.openStream (url project))] (io/copy stream (io/file zipped))) - (println "Unzipping" zipfile "to" target) - (fs-zip/unzip zipped cache))))) + (println "Unzipping" (zipfile project) "to" target) + (fs/unzip zipped target))))) -(defn build-protoc - "Compile protoc from source." +(defn uninstall + "Remove protoc if it is installed." [project] - (let [srcdir (io/file srcdir) - protoc (io/file protoc)] - (when-not (.exists protoc) - (fetch project) - (when-not (.exists (io/file srcdir "src" "protoc")) - (fs/chmod "+x" (io/file srcdir "configure")) - (fs/chmod "+x" (io/file srcdir "install-sh")) + (when (installed? project) + (let [password (read-pass) + proc (sh/proc "sudo" "-S" "make" "uninstall" + :dir (io/file (target project) (srcdir project)))] + (sh/feed-from-string proc (str password "\n")) + (sh/stream-to-out proc :out)))) + +(defn install + "Compile and install protoc to /usr/local." + [project] + (when-not (installed? project) + (fetch project) + (let [source (io/file (target project) (srcdir project))] + (when-not (.exists (io/file source "src" "protoc")) + (fs/chmod "+x" (io/file source "configure")) + (fs/chmod "+x" (io/file source "install-sh")) (println "Configuring protoc") (sh/stream-to-out (sh/proc "./configure" :dir srcdir) :out) (println "Running 'make'") @@ -137,9 +150,9 @@ (.mkdirs proto-files) (io/copy (io/file srcdir "src/google/protobuf/descriptor.proto") descriptor) - (compile-protobuf project - ["google/protobuf/descriptor.proto"] - out)))) + (protoc project + ["google/protobuf/descriptor.proto"] + (io/file target (srcdir project) "java/src/main/java"))))) (defn compile "Compile protocol buffer files located in proto dir." @@ -157,7 +170,13 @@ (compile (first args))) (apply f args))) -(defn protobuf - "Task for compiling protobuf libraries." - [project & args] - (apply compile project args)) +(defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." + :help-arglists '([subtask & args]) + :subtasks [#'install #'uninstall #'compile]} + protobuf + ([project] (println (help-for "protobuf"))) + ([project subtask & args] + (case subtask + "install" (apply install project args) + "uninstall" (apply uninstall project args) + "compile" (apply compile project args)))) From dbc311a725f2f62e859519534d731f8a4e26d125 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 17:08:40 -0700 Subject: [PATCH 277/525] doc grammar change --- lein-protobuf/src/leiningen/protobuf.clj | 27 +++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj index c12d268..99a43d4 100644 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ b/lein-protobuf/src/leiningen/protobuf.clj @@ -36,21 +36,18 @@ (second (re-matches #".*\"(.*)\".*" line)))) (defn extract-dependencies - "extract all files proto is dependent on" - [proto-path proto-file target] - (let [proto-file (io/file proto-path proto-file)] - (loop [files (vec (proto-dependencies proto-file))] - (when-not (empty? files) - (let [proto (peek files) - files (pop files)] - (if (or (.exists (io/file proto-path proto)) - (.exists (io/file target "proto" proto))) - (recur files) - (let [location (str "proto/" proto) - proto-file (io/file target location)] - (.mkdirs (.getParentFile proto-file)) - (io/copy (io/reader (io/resource location)) proto-file) - (recur (into files (proto-dependencies proto-file)))))))))) + "Extract all files proto depends on into dest." + [proto-path proto dest] + (loop [deps (proto-dependencies (io/file proto-path proto))] + (when-let [[dep & deps] (seq deps)] + (let [proto-file (io/file dest dep)] + (if (or (.exists (io/file proto-path dep)) + (.exists proto-file)) + (recur deps) + (do (.mkdirs (.getParentFile proto-file)) + (io/copy (io/reader (io/resource "proto" proto)) + proto-file) + (recur (concat deps (proto-dependencies proto-file))))))))) (defn modtime [dir] (let [files (->> dir io/file file-seq rest)] From 19ab76a0a1c9b9b9182fc260774d0676f5c06584 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 18:11:31 -0700 Subject: [PATCH 278/525] update readme --- README.md | 66 +++++++++++++++++++------------------------------------ 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index a386644..14a903c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,26 @@ -Clojure-protobuf provides a clojure interface to Google's [protocol buffers](http://code.google.com/p/protobuf). +clojure-protobuf provides a Clojure interface to Google's [protocol buffers](http://code.google.com/p/protobuf). Protocol buffers can be used to communicate with other languages over the network, and they are WAY faster to serialize and deserialize than standard Clojure objects. +## Getting started + +You'll probably want to use [Leiningen](https://github.com/technomancy/leiningen) with the +[lein-protobuf](https://github.com/flatland/lein-protobuf) plugin for compiling `.proto` files. Add +the following to your `project.clj` file: + + :dependencies [[protobuf "0.6.0"]] + :plugins [[lein-protobuf "0.1.1"]] + +Be sure to replace `"0.6.0"` and `"0.1.1"` with the latest versions listed at +http://clojars.org/protobuf and http://clojars.org/lein-protobuf. + +*Note: lein-protobuf requires at least version 2.0 of Leiningen.* + ## Usage -Write a `.proto` file: +Assuming you have the following in `resources/proto/person.proto`: -```java +```proto message Person { required int32 id = 1; required string name = 2; @@ -15,13 +29,11 @@ message Person { } ``` -If you put it in the proto directory of your project, you can compile it with [leiningen](https://github.com/technomancy/leiningen): +You can run the following to compile the `.proto` file: -``` -lein protobuf compile example.proto -``` + lein protobuf -Now you can use the protocol buffer in clojure: +Now you can use the protocol buffer in Clojure: ```clojure (use 'protobuf.core) @@ -54,7 +66,7 @@ Clojure-protobuf supports extensions to protocol buffers which provide sets and repeated fields. You can also provide metadata on protobuf fields using clojure syntax. To use these, you must import the extension file and include it when compiling. For example: -```java +```proto import "protobuf/core/extensions.proto"; message Photo { @@ -83,14 +95,7 @@ message Photo { } } ``` - -Compile the file: - -``` -lein protobuf compile example.proto -``` - -Then you can access the maps in clojure: +Then you can access the extension fields in Clojure: ```clojure (use 'protobuf) @@ -115,33 +120,6 @@ Then you can access the maps in clojure: => {:max 100.0 :min -100.0} ``` -## Installation - -You'll want to use this with the Leiningen build tool. You can get it by -putting it in your `:dependencies` and/or `:dev-dependencies`. If you -want to use the Leiningen plugin portion of clojure-protobuf, it has to -be in your dev-dependencies. - -```clojure -:dev-dependencies [[protobuf "x.x.x"]] -``` - -Replace `"x.x.x"` with the actual latest version, which you can find on -[clojars](http://clojars.org/protobuf) - -**NOTE: clojure-protobuf requires at least version 1.7.0 of Leiningen. -It will not work in earlier versions.** - -## History - -The build tool tasks provided with this library were originally for the -cake build tool. In 2011, the authors of that tool decided that their -time would be better spent working on a single, canonical build tool. -Leiningen was already the standard for Clojure, so that's where we are -now. As of version 0.6.0 (which has yet to see an final release, but is -usable), all of the cake-specific functionality has been rewritten for -Leiningen. Any version before that will not work with Leiningen. - ## Getting Help If you have any questions or need help, you can find us on IRC in [#flatland](irc://irc.freenode.net/#flatland). From cbf8d3045c15e90d4b489d9cdb9d8d55c042e9f3 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 18:24:11 -0700 Subject: [PATCH 279/525] separate lein-protobuf out into a separate repo --- lein-protobuf/project.clj | 13 -- lein-protobuf/resources/proto | 1 - lein-protobuf/src/leiningen/protobuf.clj | 179 ----------------------- 3 files changed, 193 deletions(-) delete mode 100644 lein-protobuf/project.clj delete mode 120000 lein-protobuf/resources/proto delete mode 100644 lein-protobuf/src/leiningen/protobuf.clj diff --git a/lein-protobuf/project.clj b/lein-protobuf/project.clj deleted file mode 100644 index 97e800f..0000000 --- a/lein-protobuf/project.clj +++ /dev/null @@ -1,13 +0,0 @@ -(defproject lein-protobuf "0.1.1" - :description "Leiningen plugin for clojure-protobuf." - :license {:name "Eclipse Public License" - :url "http://www.eclipse.org/legal/epl-v10.html"} - :url "https://github.com/flatland/protobuf" - :dependencies [[fs "1.2.0"] - [conch "0.2.0"]] - :eval-in-leiningen true - ;; Bug in the current 1.x branch of Leiningen causes - ;; jar to implicitly clean no matter what, wiping stuff. - ;; This prevents that. - :disable-implicit-clean true - :checksum-deps true) diff --git a/lein-protobuf/resources/proto b/lein-protobuf/resources/proto deleted file mode 120000 index b943f93..0000000 --- a/lein-protobuf/resources/proto +++ /dev/null @@ -1 +0,0 @@ -../../proto \ No newline at end of file diff --git a/lein-protobuf/src/leiningen/protobuf.clj b/lein-protobuf/src/leiningen/protobuf.clj deleted file mode 100644 index 99a43d4..0000000 --- a/lein-protobuf/src/leiningen/protobuf.clj +++ /dev/null @@ -1,179 +0,0 @@ -(ns leiningen.protobuf - (:refer-clojure :exclude [compile]) - (:use [clojure.string :only [join]] - [leiningen.help :only [help-for]] - [leiningen.javac :only [javac]] - [leiningen.core.eval :only [get-os]] - [leiningen.core.user :only [leiningen-home]] - [robert.hooke :only [add-hook]]) - (:require [clojure.java.io :as io] - [fs.core :as fs] - [fs.compression :as fs-zip] - [conch.core :as sh]) - (:import java.util.zip.ZipFile)) - -(def default-version "2.3.0") -(defn version - [project] - (or (:protobuf-version project) "default-version")) -(defn srcdir [project] (str "protobuf-" (version project))) -(defn zipfile [project] (format "protobuf-%s.zip" (version project))) - -(def ^{:dynamic true} *compile?* true) - -(defn url [project] - (java.net.URL. - (format "http://protobuf.googlecode.com/files/protobuf-%s.zip" (version project)))) - -(defn target [project] - (doto (io/file (:target-path project)) - .mkdirs)) - -(defn- proto-dependencies - "look for lines starting with import in proto-file" - [proto-file] - (for [line (line-seq (io/reader proto-file)) :when (.startsWith line "import")] - (second (re-matches #".*\"(.*)\".*" line)))) - -(defn extract-dependencies - "Extract all files proto depends on into dest." - [proto-path proto dest] - (loop [deps (proto-dependencies (io/file proto-path proto))] - (when-let [[dep & deps] (seq deps)] - (let [proto-file (io/file dest dep)] - (if (or (.exists (io/file proto-path dep)) - (.exists proto-file)) - (recur deps) - (do (.mkdirs (.getParentFile proto-file)) - (io/copy (io/reader (io/resource "proto" proto)) - proto-file) - (recur (concat deps (proto-dependencies proto-file))))))))) - -(defn modtime [dir] - (let [files (->> dir io/file file-seq rest)] - (if (empty? files) - 0 - (apply max (map fs/mod-time files))))) - -(defn proto-file? [file] - (let [name (.getName file)] - (and (.endsWith name ".proto") - (not (.startsWith name "."))))) - -(defn proto-files [dir] - (for [file (rest (file-seq dir)) :when (proto-file? file)] - (.substring (.getPath file) (inc (count (.getPath dir)))))) - -(defn installed? [project] - (try (.contains (sh/stream-to-string (sh/proc "protoc" "--version") :out) (version project)) - (catch java.io.IOException e))) - -(defn read-pass [] - (print "Password: ") - (flush) - (join (.readPassword (System/console)))) - -(defn fetch - "Fetch protocol-buffer source and unzip it." - [project] - (let [target (target project)] - (when-not (.exists (io/file target (srcdir project))) - (let [zipped (io/file target (zipfile project))] - (println "Downloading" (zipfile project)) - (with-open [stream (.openStream (url project))] - (io/copy stream (io/file zipped))) - (println "Unzipping" (zipfile project) "to" target) - (fs/unzip zipped target))))) - -(defn uninstall - "Remove protoc if it is installed." - [project] - (when (installed? project) - (let [password (read-pass) - proc (sh/proc "sudo" "-S" "make" "uninstall" - :dir (io/file (target project) (srcdir project)))] - (sh/feed-from-string proc (str password "\n")) - (sh/stream-to-out proc :out)))) - -(defn install - "Compile and install protoc to /usr/local." - [project] - (when-not (installed? project) - (fetch project) - (let [source (io/file (target project) (srcdir project))] - (when-not (.exists (io/file source "src" "protoc")) - (fs/chmod "+x" (io/file source "configure")) - (fs/chmod "+x" (io/file source "install-sh")) - (println "Configuring protoc") - (sh/stream-to-out (sh/proc "./configure" :dir srcdir) :out) - (println "Running 'make'") - (sh/stream-to-out (sh/proc "make" :dir srcdir) :out))))) - -(defn compile-protobuf - "Create .java and .class files from the provided .proto files." - ([project protos] - (compile-protobuf project protos (io/file (target project) "protosrc"))) - ([project protos dest] - (let [target (target project) - dest (.getAbsoluteFile dest) - dest-path (.getPath dest) - proto-path (.getAbsoluteFile (io/file (or (:proto-path project) "proto")))] - (when (or (> (modtime proto-path) (modtime dest)) - (> (modtime proto-path) (modtime (str target "/classes")))) - (.mkdirs dest) - (.mkdirs proto-path) - (doseq [proto protos] - (extract-dependencies proto-path proto target) - (let [args [protoc proto (str "--java_out=" dest-path) "-I." - (str "-I" (.getAbsoluteFile (io/file target "proto"))) - (str "-I" proto-path)]] - (println " > " (join " " args)) - (let [protoc-result (apply sh/proc (concat args [:dir proto-path]))] - (if (not (= (sh/exit-code protoc-result) 0)) - (println "ERROR: " (sh/stream-to-string protoc-result :err)))))) - (binding [*compile?* false] - (javac (assoc project :java-source-paths [dest-path]))))))) - -(defn compile-google-protobuf - "Compile com.google.protobuf.*" - [project] - (let [proto-files (io/file (get project :proto-path "proto") "google/protobuf") - target (target project) - descriptor (io/file proto-files "descriptor.proto") - out (io/file srcdir "java/src/main/java")] - (when-not (and (.exists descriptor) - (.exists (io/file out "com/google/protobuf/DescriptorProtos.java"))) - (fetch project) - (.mkdirs proto-files) - (io/copy (io/file srcdir "src/google/protobuf/descriptor.proto") - descriptor) - (protoc project - ["google/protobuf/descriptor.proto"] - (io/file target (srcdir project) "java/src/main/java"))))) - -(defn compile - "Compile protocol buffer files located in proto dir." - ([project] - (apply compile project (proto-files (io/file (or (:proto-path project) "proto"))))) - ([project & files] - (build-protoc project) - (when (and (= "protobuf" (:name project))) - (compile-google-protobuf project)) - (compile-protobuf project files))) - -(add-hook #'javac - (fn [f & args] - (when *compile?* - (compile (first args))) - (apply f args))) - -(defn ^{:doc "Tasks for installing and uninstalling protobuf libraries." - :help-arglists '([subtask & args]) - :subtasks [#'install #'uninstall #'compile]} - protobuf - ([project] (println (help-for "protobuf"))) - ([project subtask & args] - (case subtask - "install" (apply install project args) - "uninstall" (apply uninstall project args) - "compile" (apply compile project args)))) From cd5a646594b75b319428cb0c7ba2c5adf2f4c022 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 18:25:53 -0700 Subject: [PATCH 280/525] example.proto is at resources/proto/protobuf/test/example.proto --- examples/.gitignore | 2 -- examples/example.proto | 28 ---------------------------- 2 files changed, 30 deletions(-) delete mode 100644 examples/.gitignore delete mode 100644 examples/example.proto diff --git a/examples/.gitignore b/examples/.gitignore deleted file mode 100644 index 6a70a53..0000000 --- a/examples/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.class -*.java \ No newline at end of file diff --git a/examples/example.proto b/examples/example.proto deleted file mode 100644 index ca639ac..0000000 --- a/examples/example.proto +++ /dev/null @@ -1,28 +0,0 @@ -import "collections.proto"; - -message Photo { - required int32 id = 1; - required string path = 2; - repeated string labels = 3 [(set) = true]; - repeated Attr attrs = 4 [(map) = true]; - repeated Tag tags = 5 [(map_by) = "person_id"]; - - message Attr { - required string key = 1; - optional string val = 2; - } - - message Tag { - required int32 person_id = 1; - optional int32 x_coord = 2; - optional int32 y_coord = 3; - optional int32 width = 4; - optional int32 height = 5; - } -} - -message Person { - required int32 id = 1; - required string name = 2; - optional string email = 3; -} From f453ef0c8739c21ff2b538a03e7c95cdfb34838a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 18:32:50 -0700 Subject: [PATCH 281/525] 0.6.1-beta1 --- project.clj | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 1b2f804..1739915 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.0-beta19" +(defproject protobuf "0.6.1-beta1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,13 +7,9 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.1.1"]] + :plugins [[lein-protobuf "0.2.0-beta1"]] :profiles {:dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} :hooks [leiningen.protobuf] - ;; Bug in the current 1.x branch of Leiningen causes - ;; jar to implicitly clean no matter what, wiping stuff. - ;; This prevents that. - :disable-implicit-clean true :checksum-deps true :java-source-paths ["src"]) From bdcc3e4099b316d5989b0ad3b6fbd7e9c7956548 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 16 Jul 2012 19:09:34 -0700 Subject: [PATCH 282/525] use protobuf 2.4.1 --- project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/project.clj b/project.clj index 1739915..5767016 100644 --- a/project.clj +++ b/project.clj @@ -10,6 +10,7 @@ :plugins [[lein-protobuf "0.2.0-beta1"]] :profiles {:dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} + :protobuf-version "2.4.1" :hooks [leiningen.protobuf] :checksum-deps true :java-source-paths ["src"]) From cc24481e463deacaeee8084494f259a2d5755c1a Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 18 Jul 2012 14:56:26 -0700 Subject: [PATCH 283/525] update lein-protobuf --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 5767016..547f1f9 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.1-beta1" +(defproject protobuf "0.6.1-beta2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta1"]] + :plugins [[lein-protobuf "0.2.0-beta2"]] :profiles {:dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} :protobuf-version "2.4.1" From c41c9113161f6b1a3e36dacd3f4b12e60c9e3ead Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 24 Jul 2012 17:34:54 -0500 Subject: [PATCH 284/525] Test against multiple Clojure versions. --- project.clj | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 547f1f9..3ada145 100644 --- a/project.clj +++ b/project.clj @@ -7,9 +7,15 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta2"]] - :profiles {:dev {:dependencies [[gloss "0.2.1"] + :plugins [[lein-protobuf "0.2.0-beta3"]] + :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} + :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} + :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} + :dev {:dependencies [[gloss "0.2.1"] [io "0.2.0-beta2"]]}} + :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" + :snapshots true + :releases {:checksum :fail :update :always}}} :protobuf-version "2.4.1" :hooks [leiningen.protobuf] :checksum-deps true From 8523115400314c9dbd619e7ce43972c8ae86cfa6 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 24 Jul 2012 17:35:16 -0500 Subject: [PATCH 285/525] Add travis file. --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..65a5762 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: clojure +lein: lein2 +script: lein2 testall \ No newline at end of file From b6b1180643f521bfe49660de05ed12a661264480 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 13 Aug 2012 12:40:39 -0700 Subject: [PATCH 286/525] ignore pom.xml.asc --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9468b9f..ba264f3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ lib classes build pom.xml +pom.xml.asc *.jar *.class proto/google/protobuf/descriptor.proto From bebad893eda3ea473ceda9424c959df3bcc0c585 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 13 Aug 2012 12:40:52 -0700 Subject: [PATCH 287/525] update to lein-protobuf with auto-hooks --- project.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 3ada145..d303747 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta3"]] + :plugins [[lein-protobuf "0.2.0-beta5"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} @@ -17,6 +17,5 @@ :snapshots true :releases {:checksum :fail :update :always}}} :protobuf-version "2.4.1" - :hooks [leiningen.protobuf] :checksum-deps true :java-source-paths ["src"]) From eb78ba5b7c2bc420cfa6b36b8993755f6ce6ae9e Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 13 Aug 2012 16:18:52 -0500 Subject: [PATCH 288/525] Update to lein-protobuf 0.2.0-beta6. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index d303747..4e3368a 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta5"]] + :plugins [[lein-protobuf "0.2.0-beta6"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} From d823f2262bb32ae71956888b297db4c2da76c7bb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 14 Aug 2012 16:10:20 -0700 Subject: [PATCH 289/525] bump protobuf default version to 2.4.1 --- project.clj | 1 - 1 file changed, 1 deletion(-) diff --git a/project.clj b/project.clj index 4e3368a..df6367d 100644 --- a/project.clj +++ b/project.clj @@ -16,6 +16,5 @@ :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} - :protobuf-version "2.4.1" :checksum-deps true :java-source-paths ["src"]) From 62bd5ffc84ac1963887c25dd92497c9e41a3238b Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Tue, 14 Aug 2012 18:20:05 -0500 Subject: [PATCH 290/525] Bump lein-protobuf version. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index df6367d..5fd707e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.1-beta2" +(defproject protobuf "0.6.1-beta3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.2-alpha1"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta6"]] + :plugins [[lein-protobuf "0.2.0-beta7"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} From ecd9c20987d88bb81f7a20d7cbc60bc854b0f109 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Wed, 22 Aug 2012 20:44:24 -0500 Subject: [PATCH 291/525] Update versions. --- project.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index 5fd707e..478dce7 100644 --- a/project.clj +++ b/project.clj @@ -1,18 +1,18 @@ -(defproject protobuf "0.6.1-beta3" +(defproject protobuf "0.6.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/flatland/clojure-protobuf" :dependencies [[org.clojure/clojure "1.4.0"] [ordered-collections "0.4.0"] - [useful "0.8.2-alpha1"] + [useful "0.8.4"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0-beta7"]] + :plugins [[lein-protobuf "0.2.0"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} :dev {:dependencies [[gloss "0.2.1"] - [io "0.2.0-beta2"]]}} + [io "0.2.1"]]}} :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} From 431c8bd1aa079519c37f8482832b01e604d826ea Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 7 Aug 2012 14:06:27 -0700 Subject: [PATCH 292/525] This may as well be public. Package-private is just stupid anyway. --- src/protobuf/core/PersistentProtocolBufferMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index bc6aa94..f43679f 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -114,7 +114,7 @@ public String toString() { } }; - final Descriptors.Descriptor type; + public final Descriptors.Descriptor type; public final NamingStrategy namingStrategy; public static final Object NULL = new Object(); From d85b07b52de963092072aa39ab5479ac4ff8faef Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 7 Aug 2012 14:06:40 -0700 Subject: [PATCH 293/525] Refuse to serialize with non-empty extmap --- src/protobuf/core/PersistentProtocolBufferMap.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index f43679f..1f9b258 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -314,6 +314,9 @@ protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, D } public byte[] toByteArray() { + if (RT.count(ext) != 0) { + throw new IllegalStateException("Can't serialize unexpected keys: " + RT.printString(RT.keys(ext))); + } return message().toByteArray(); } From 8e7157ecf28706c5a3d20b07b6ae71a4b3718ff1 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Tue, 7 Aug 2012 16:17:21 -0700 Subject: [PATCH 294/525] Make sure conj and construct use the extmap. --- .../core/PersistentProtocolBufferMap.java | 42 +++++++++++-------- test/protobuf/core_test.clj | 12 +++++- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index 1f9b258..f41aa06 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -273,7 +273,7 @@ static public PersistentProtocolBufferMap parseDelimitedFrom(Def def, InputStrea static public PersistentProtocolBufferMap construct(Def def, Object keyvals) { PersistentProtocolBufferMap protobuf = new PersistentProtocolBufferMap(null, def); - return (PersistentProtocolBufferMap)protobuf.cons(keyvals); + return protobuf.cons(keyvals); } protected PersistentProtocolBufferMap(IPersistentMap meta, Def def) { @@ -557,16 +557,17 @@ protected void setField(DynamicMessage.Builder builder, Descriptors.FieldDescrip } } - protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object key, Object value) { + // returns true if the protobuf can store this key + protected boolean addField(DynamicMessage.Builder builder, Object key, Object value) { if (key == null) { - return builder; + return false; } Descriptors.FieldDescriptor field = def.fieldDescriptor(key); if (field == null) { - return builder; + return false; } if (value == null && !(field.getOptions().getExtension(Extensions.nullable))) { - return builder; + return true; } boolean set = field.getOptions().getExtension(Extensions.set); @@ -621,7 +622,7 @@ protected DynamicMessage.Builder addField(DynamicMessage.Builder builder, Object setField(builder, field, v); } - return builder; + return true; } @Override @@ -686,18 +687,18 @@ public Object getValAt(Object key, Object notFound, boolean use_extensions) { } @Override - public IPersistentMap assoc(Object key, Object value) { - Descriptors.FieldDescriptor field = def.fieldDescriptor(key); + public PersistentProtocolBufferMap assoc(Object key, Object value) { + DynamicMessage.Builder builder = builder(); - if (field != null) { - return new PersistentProtocolBufferMap(meta(), ext, def, addField(builder(), field, value)); + if (addField(builder, key, value)) { + return new PersistentProtocolBufferMap(meta(), ext, def, builder); } else { - return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), def, builder()); + return new PersistentProtocolBufferMap(meta(), (IPersistentMap)RT.assoc(ext, key, value), def, builder); } } @Override - public IPersistentMap assocEx(Object key, Object value) { + public PersistentProtocolBufferMap assocEx(Object key, Object value) { if (containsKey(key)) { throw new RuntimeException("Key already present"); } @@ -705,24 +706,29 @@ public IPersistentMap assocEx(Object key, Object value) { } @Override - public IPersistentCollection cons(Object o) { - DynamicMessage.Builder builder = builder(); + public PersistentProtocolBufferMap cons(Object o) { if (o instanceof Map.Entry) { Map.Entry e = (Map.Entry)o; - addField(builder, e.getKey(), e.getValue()); + return assoc(e.getKey(), e.getValue()); } else if (o instanceof IPersistentVector) { IPersistentVector v = (IPersistentVector)o; if (v.count() != 2) { throw new IllegalArgumentException("Vector arg to map conj must be a pair"); } - addField(builder, v.nth(0), v.nth(1)); + return assoc(v.nth(0), v.nth(1)); } else { + DynamicMessage.Builder builder = builder(); + IPersistentMap ext = this.ext; for (ISeq s = RT.seq(o); s != null; s = s.next()) { Map.Entry e = (Map.Entry)s.first(); - addField(builder, e.getKey(), e.getValue()); + + Object k = e.getKey(), v = e.getValue(); + if (!addField(builder, k, v)) { + ext = (IPersistentMap)RT.assoc(ext, k, v); + } } + return new PersistentProtocolBufferMap(meta(), ext, def, builder); } - return new PersistentProtocolBufferMap(meta(), def, builder); } public PersistentProtocolBufferMap append(IPersistentMap map) { diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index dd73e6e..bde0cf6 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -116,7 +116,17 @@ (is (= p2 m2)) (is (= m2 p2)) (is (= (into {} m2) (into {} p2))) - (is (= (set (keys m2)) (set (keys p2)))))) + (is (= (set (keys m2)) (set (keys p2))))) + + (let [m {:id 5 :wat 10} + p (protobuf Foo m)] + (testing "protobuf function uses extmap" + (is (= (:wat m) (:wat p))) + (let [p (conj p {:stuff 15})] + (is (= 15 (:stuff p))))) + + (is (thrown? Exception (protobuf-dump p)) + "Should refuse to serialize with stuff in extmap"))) (deftest test-string-keys (let [p (protobuf Foo "id" 5 "label" "rad")] From a0557dcf0e019de85109f0f5c58e29847c57cdc4 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 13 Sep 2012 17:28:03 -0700 Subject: [PATCH 295/525] Add a size-limit argument to defs. Needed for allowing to read large protobufs. --- src/protobuf/core.clj | 10 ++-- .../core/PersistentProtocolBufferMap.java | 54 +++++++++++-------- test/protobuf/core_test.clj | 2 +- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/protobuf/core.clj b/src/protobuf/core.clj index 206f610..e5a6203 100644 --- a/src/protobuf/core.clj +++ b/src/protobuf/core.clj @@ -24,13 +24,17 @@ (if (or (protodef? def) (nil? def)) def (protodef def PersistentProtocolBufferMap$Def/convertUnderscores))) - ([def ^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy] + ([def opts] (when def - (let [^Descriptors$Descriptor descriptor + (let [{:keys [^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy + size-limit] + :or {naming-strategy PersistentProtocolBufferMap$Def/convertUnderscores + size-limit 67108864}} opts ;; 64MiB + ^Descriptors$Descriptor descriptor (if (instance? Descriptors$Descriptor def) def (Reflector/invokeStaticMethod ^Class def "getDescriptor" (to-array nil)))] - (PersistentProtocolBufferMap$Def/create descriptor naming-strategy))))) + (PersistentProtocolBufferMap$Def/create descriptor naming-strategy size-limit))))) (defn protobuf "Construct a protobuf of the given type." diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index f41aa06..d7317f2 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -116,43 +116,52 @@ public String toString() { public final Descriptors.Descriptor type; public final NamingStrategy namingStrategy; + public final int sizeLimit; public static final Object NULL = new Object(); // keys should be FieldDescriptors, except that NULL is used as a replacement for real null ConcurrentHashMap key_to_field; - static ConcurrentHashMap> type_to_def = new ConcurrentHashMap>(); + private static final class DefOptions { + public final Descriptors.Descriptor type; + public final NamingStrategy strat; + public final int sizeLimit; + public DefOptions(Descriptors.Descriptor type, NamingStrategy strat, int sizeLimit) { + this.type = type; + this.strat = strat; + this.sizeLimit = sizeLimit; + } - public static Def create(Class c, NamingStrategy strat) throws NoSuchMethodException, - IllegalAccessException, InvocationTargetException { - Descriptors.Descriptor type = (Descriptors.Descriptor)c.getMethod("getDescriptor").invoke( - null); - return create(type, strat); - } + public boolean equals(Object other) { + if (this.getClass() != other.getClass()) + return false; + DefOptions od = (DefOptions)other; + return type.equals(od.type) && strat.equals(od.strat) && sizeLimit == od.sizeLimit; + } - public static Def create(Descriptors.Descriptor type, NamingStrategy strat) { - ConcurrentHashMap defCache = type_to_def.get(strat); - if (defCache == null) { - defCache = new ConcurrentHashMap(); - ConcurrentHashMap previous = type_to_def.putIfAbsent(strat, - defCache); - if (previous != null) { - defCache = previous; - } + public int hashCode() { + return type.hashCode() + strat.hashCode() + sizeLimit; } + } + + static ConcurrentHashMap defCache = new ConcurrentHashMap(); + + public static Def create(Descriptors.Descriptor type, NamingStrategy strat, int sizeLimit) { + DefOptions opts = new DefOptions(type, strat, sizeLimit); Def def = defCache.get(type); if (def == null) { - def = new Def(type, strat); - defCache.putIfAbsent(type, def); + def = new Def(type, strat, sizeLimit); + defCache.putIfAbsent(opts, def); } return def; } - protected Def(Descriptors.Descriptor type, NamingStrategy strat) { + protected Def(Descriptors.Descriptor type, NamingStrategy strat, int sizeLimit) { this.type = type; this.key_to_field = new ConcurrentHashMap(); this.namingStrategy = strat; + this.sizeLimit = sizeLimit; } public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferException { @@ -160,6 +169,7 @@ public DynamicMessage parseFrom(byte[] bytes) throws InvalidProtocolBufferExcept } public DynamicMessage parseFrom(CodedInputStream input) throws IOException { + input.setSizeLimit(sizeLimit); return DynamicMessage.parseFrom(type, input); } @@ -447,7 +457,8 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, } case MESSAGE: Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), - this.def.namingStrategy); + this.def.namingStrategy, + this.def.sizeLimit); DynamicMessage message = (DynamicMessage)value; // Total hack because getField() doesn't return an empty array for repeated messages. @@ -506,7 +517,8 @@ protected Object toProtoValue(Descriptors.FieldDescriptor field, Object value) { protobuf = (PersistentProtocolBufferMap)value; } else { Def fieldDef = PersistentProtocolBufferMap.Def.create(field.getMessageType(), - this.def.namingStrategy); + this.def.namingStrategy, + this.def.sizeLimit); protobuf = PersistentProtocolBufferMap.construct(fieldDef, value); } return protobuf.message(); diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index bde0cf6..e888ccd 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -7,7 +7,7 @@ (def Foo (protodef protobuf.test.Core$Foo)) (def FooUnder (protodef protobuf.test.Core$Foo - protobuf.core.PersistentProtocolBufferMap$Def/protobufNames)) + {:naming-strategy protobuf.core.PersistentProtocolBufferMap$Def/protobufNames})) (def Bar (protodef protobuf.test.Core$Bar)) (def Response (protodef protobuf.test.Core$Response)) (def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) From 06103b5b17b9c56ff0cc8b467ccece5a0f8cd4b8 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 13 Sep 2012 17:40:10 -0700 Subject: [PATCH 296/525] 0.6.2-alpha1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 478dce7..3f75543 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.1" +(defproject protobuf "0.6.2-alpha1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 8dc560fa3f6223a76814ab64e26e0a8c31b77107 Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Fri, 14 Sep 2012 15:13:06 -0500 Subject: [PATCH 297/525] Update lein-protobuf. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 3f75543..b4bad7f 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.4"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.0"]] + :plugins [[lein-protobuf "0.2.1"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} From 8d63a2985bff55cc07740c6c436bba3b8af221b4 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 25 Sep 2012 15:34:01 -0700 Subject: [PATCH 298/525] Revert "Refuse to serialize with non-empty extmap" We need to take another pass at this once we can enforce this for sub-maps. This reverts commit c5671cec333a5c412ac13913df680aecbd12dd68. --- src/protobuf/core/PersistentProtocolBufferMap.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/protobuf/core/PersistentProtocolBufferMap.java index d7317f2..af6f33e 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/protobuf/core/PersistentProtocolBufferMap.java @@ -324,9 +324,6 @@ protected PersistentProtocolBufferMap(IPersistentMap meta, IPersistentMap ext, D } public byte[] toByteArray() { - if (RT.count(ext) != 0) { - throw new IllegalStateException("Can't serialize unexpected keys: " + RT.printString(RT.keys(ext))); - } return message().toByteArray(); } From f2647ab221e2069c9000494c2452bfea3f9b1355 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 25 Sep 2012 15:35:29 -0700 Subject: [PATCH 299/525] 0.6.2-alpha2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index b4bad7f..3860962 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.2-alpha1" +(defproject protobuf "0.6.2-alpha2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From dd588e52f971477eafeecbfa2103e5ff7e62cefb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 10 Oct 2012 10:28:50 -0700 Subject: [PATCH 300/525] disable test until we re-enable extmap serialization check --- test/protobuf/core_test.clj | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/protobuf/core_test.clj b/test/protobuf/core_test.clj index e888ccd..d896c33 100644 --- a/test/protobuf/core_test.clj +++ b/test/protobuf/core_test.clj @@ -125,8 +125,10 @@ (let [p (conj p {:stuff 15})] (is (= 15 (:stuff p))))) - (is (thrown? Exception (protobuf-dump p)) - "Should refuse to serialize with stuff in extmap"))) + ;; TODO add test back once we re-enable this check + (comment + (is (thrown? Exception (protobuf-dump p)) + "Should refuse to serialize with stuff in extmap")))) (deftest test-string-keys (let [p (protobuf Foo "id" 5 "label" "rad")] From ca63ee195f8cb0fc6afaa7a0a1dffa3496ba6cd4 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 10 Oct 2012 10:32:30 -0700 Subject: [PATCH 301/525] compile for java 5 --- project.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 3860962..b4c8779 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.2-alpha2" +(defproject protobuf "0.6.2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -8,6 +8,7 @@ [useful "0.8.4"] [schematic "0.0.5"]] :plugins [[lein-protobuf "0.2.1"]] + :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} From 33238e744e5b73273539288a47ceb11fa8b14bed Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 10 Oct 2012 10:32:54 -0700 Subject: [PATCH 302/525] 0.6.2 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index b4c8779..7b8989a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.2-SNAPSHOT" +(defproject protobuf "0.6.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From e0bdf4ed1ffa45d325b472908a4b6e44743ebeec Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 10 Oct 2012 10:35:25 -0700 Subject: [PATCH 303/525] snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 7b8989a..eced9ff 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject protobuf "0.6.2" +(defproject protobuf "0.6.3-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From f5870b933498a882f4f3c41ec6f458b1ff40dc60 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 14 Nov 2012 15:55:01 -0800 Subject: [PATCH 304/525] update to new lein-protobuf --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index eced9ff..b637214 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ [ordered-collections "0.4.0"] [useful "0.8.4"] [schematic "0.0.5"]] - :plugins [[lein-protobuf "0.2.1"]] + :plugins [[lein-protobuf "0.2.3-SNAPSHOT"]] :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} From a48d0c2bfd138a9c858ba89bb363f72ac36f052b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 14 Nov 2012 16:24:14 -0800 Subject: [PATCH 305/525] move namespaces under flatland --- project.clj | 10 +- .../proto/flatland/protobuf/extensions.proto | 20 ++++ .../proto/flatland/protobuf/test/codec.proto | 47 +++++++++ .../proto/flatland/protobuf/test/core.proto | 97 +++++++++++++++++++ .../flatland/protobuf/test/example.proto | 32 ++++++ .../PersistentProtocolBufferMap.java | 2 +- src/{ => flatland}/protobuf/codec.clj | 12 +-- src/{ => flatland}/protobuf/core.clj | 14 +-- src/{ => flatland}/protobuf/schema.clj | 8 +- test/{ => flatland}/protobuf/codec_test.clj | 10 +- test/{ => flatland}/protobuf/core_test.clj | 44 ++++----- test/{ => flatland}/protobuf/example_test.clj | 6 +- 12 files changed, 250 insertions(+), 52 deletions(-) create mode 100644 resources/proto/flatland/protobuf/extensions.proto create mode 100644 resources/proto/flatland/protobuf/test/codec.proto create mode 100644 resources/proto/flatland/protobuf/test/core.proto create mode 100644 resources/proto/flatland/protobuf/test/example.proto rename src/{protobuf/core => flatland/protobuf}/PersistentProtocolBufferMap.java (99%) rename src/{ => flatland}/protobuf/codec.clj (88%) rename src/{ => flatland}/protobuf/core.clj (90%) rename src/{ => flatland}/protobuf/schema.clj (94%) rename test/{ => flatland}/protobuf/codec_test.clj (89%) rename test/{ => flatland}/protobuf/core_test.clj (91%) rename test/{ => flatland}/protobuf/example_test.clj (70%) diff --git a/project.clj b/project.clj index b637214..20f79d9 100644 --- a/project.clj +++ b/project.clj @@ -1,19 +1,19 @@ -(defproject protobuf "0.6.3-SNAPSHOT" +(defproject org.flatland/protobuf "0.6.3-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/flatland/clojure-protobuf" :dependencies [[org.clojure/clojure "1.4.0"] - [ordered-collections "0.4.0"] - [useful "0.8.4"] - [schematic "0.0.5"]] + [org.flatland/useful "0.8.9-SNAPSHOT"] + [org.flatland/schematic "0.1.0-SNAPSHOT"] + [ordered-collections "0.4.0"]] :plugins [[lein-protobuf "0.2.3-SNAPSHOT"]] :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} :dev {:dependencies [[gloss "0.2.1"] - [io "0.2.1"]]}} + [org.flatland/io "0.2.2-SNAPSHOT"]]}} :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} diff --git a/resources/proto/flatland/protobuf/extensions.proto b/resources/proto/flatland/protobuf/extensions.proto new file mode 100644 index 0000000..8fb857e --- /dev/null +++ b/resources/proto/flatland/protobuf/extensions.proto @@ -0,0 +1,20 @@ +import "google/protobuf/descriptor.proto"; + +option java_package = "flatland.protobuf"; +option java_outer_classname = "Extensions"; + +extend google.protobuf.FieldOptions { + optional bool set = 52001; + optional bool map = 52002; + optional string map_by = 52003; + optional bool counter = 52004; + optional bool succession = 52005; + optional string meta = 52010; + optional bool nullable = 52020; + optional string null_string = 52021; + optional sint32 null_int = 52022; + optional sint64 null_long = 52023; + optional float null_float = 52024; + optional double null_double = 52025; + optional uint32 null_enum = 52026; +} diff --git a/resources/proto/flatland/protobuf/test/codec.proto b/resources/proto/flatland/protobuf/test/codec.proto new file mode 100644 index 0000000..02b6635 --- /dev/null +++ b/resources/proto/flatland/protobuf/test/codec.proto @@ -0,0 +1,47 @@ +package flatland.protobuf.test.codec; + +import "flatland/protobuf/extensions.proto"; + +option java_package = "flatland.protobuf.test"; +option java_outer_classname = "Codec"; + +message Foo { + optional int32 foo = 1; + optional int32 bar = 2; + optional int32 baz = 3; + + repeated string tags = 4; + repeated Item tag_set = 5 [(set) = true]; + repeated Entry num_map = 6 [(map) = true]; + + optional Foo nested = 7; + + repeated int32 revisions = 14; + optional fixed32 proto_length = 15; +} + +message Item { + required string item = 1; + required bool exists = 2 [default = true]; +} + +message Entry { + required int32 key = 1; + required string val = 2; +} + +message Edge { + required string to_id = 1; + optional string a = 2; + optional string b = 3; + optional bool deleted = 4; +} + +message Node { + optional string id = 1; + repeated Edge edges = 2 [(map_by) = "to_id"]; + optional int32 rev = 3; + optional int32 foo = 4; + optional string bar = 5; + repeated int32 baz = 6; +} \ No newline at end of file diff --git a/resources/proto/flatland/protobuf/test/core.proto b/resources/proto/flatland/protobuf/test/core.proto new file mode 100644 index 0000000..4ef5398 --- /dev/null +++ b/resources/proto/flatland/protobuf/test/core.proto @@ -0,0 +1,97 @@ +package flatland.protobuf.test.core; + +import "flatland/protobuf/extensions.proto"; + +option java_package = "flatland.protobuf.test"; +option java_outer_classname = "Core"; + +message Foo { + optional uint32 id = 1 [default = 43]; + optional string label = 2 [(meta) = "{:a 1 :b 2 :c 3}"]; + repeated string tags = 3; + optional Foo parent = 4; + repeated Response responses = 5; + + repeated double doubles = 6; + repeated float floats = 7; + + optional double lat = 8; + optional float long = 9; + + repeated Count counts = 10 [(map_by) = "key"]; + + repeated Time time = 11 [(succession) = true]; + + enum Response { + yes = 0; + no = 1; + maybe = 2; + not_sure = 3; + } + + repeated Item tag_set = 20 [(set) = true]; + repeated Pair attr_map = 21 [(map) = true]; + repeated Foo foo_by_id = 22 [(map_by) = "id"]; + repeated Group groups = 23 [(map) = true]; + repeated Item item_map = 24 [(map_by) = "item"]; + repeated Pair pair_map = 25 [(map_by) = "key"]; + optional bool deleted = 26 [default = false]; +} + +message Bar { + optional int32 int = 1 [(nullable) = true, (null_int) = -1]; + optional int64 long = 2 [(nullable) = true, (null_long) = -999999999999]; + optional float flt = 3 [(nullable) = true, (null_float) = -0.0001]; + optional double dbl = 4 [(nullable) = true, (null_double) = -0.00000001]; + optional string str = 5 [(nullable) = true, (null_string) = "NULL"]; + optional Enu enu = 6 [(nullable) = true, (null_enum) = 3]; + + enum Enu { + a = 0; + b = 1; + c = 2; + nil = 3; + } + + repeated string label = 10 [(nullable) = true, (null_string) = "", (succession) = true]; + repeated string labels = 11 [(nullable) = true, (null_string) = ""]; +} + +message Time { + optional sint32 year = 1; + optional int32 month = 2; + optional int32 day = 3; + optional int32 hour = 4; + optional int32 minute = 5; +} + +message Pair { + required string key = 1; + required string val = 2; +} + +message Group { + required string key = 1; + repeated Foo val = 2; +} + +message Item { + required string item = 1; + required bool exists = 2 [default = true]; +} + +message Count { + required string key = 1; + repeated int32 i = 2 [(counter) = true]; + repeated double d = 3 [(counter) = true]; +} + +message ErrorMsg { + required sint32 code = 1; + optional string data = 2; +} + +message Response { + required bool ok = 1; + optional ErrorMsg error = 2; +} \ No newline at end of file diff --git a/resources/proto/flatland/protobuf/test/example.proto b/resources/proto/flatland/protobuf/test/example.proto new file mode 100644 index 0000000..fb8067b --- /dev/null +++ b/resources/proto/flatland/protobuf/test/example.proto @@ -0,0 +1,32 @@ +package flatland.protobuf.test.example; + +import "flatland/protobuf/extensions.proto"; + +option java_package = "flatland.protobuf.test"; +option java_outer_classname = "Example"; + +message Photo { + required int32 id = 1; + required string path = 2; + repeated Label labels = 3 [(set) = true]; + repeated Attr attrs = 4 [(map) = true]; + repeated Tag tags = 5 [(map_by) = "person_id"]; + + message Label { + required string item = 1; + required bool exists = 2; + } + + message Attr { + required string key = 1; + optional string val = 2; + } + + message Tag { + required int32 person_id = 1; + optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; + optional int32 y_coord = 3; + optional int32 width = 4; + optional int32 height = 5; + } +} diff --git a/src/protobuf/core/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java similarity index 99% rename from src/protobuf/core/PersistentProtocolBufferMap.java rename to src/flatland/protobuf/PersistentProtocolBufferMap.java index af6f33e..b296af0 100644 --- a/src/protobuf/core/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -8,7 +8,7 @@ * You must not remove this notice, or any other, from this software. **/ -package protobuf.core; +package flatland.protobuf; import java.io.IOException; import java.io.InputStream; diff --git a/src/protobuf/codec.clj b/src/flatland/protobuf/codec.clj similarity index 88% rename from src/protobuf/codec.clj rename to src/flatland/protobuf/codec.clj index 0d26d6b..01baecc 100644 --- a/src/protobuf/codec.clj +++ b/src/flatland/protobuf/codec.clj @@ -1,12 +1,12 @@ -(ns protobuf.codec - (:use protobuf.core +(ns flatland.protobuf.codec + (:use flatland.protobuf.core [gloss.core.protocols :only [Reader Writer]] [gloss.core.formats :only [to-buf-seq]] - [useful.fn :only [fix]] - [useful.experimental :only [lift-meta]] + [flatland.useful.fn :only [fix]] + [flatland.useful.experimental :only [lift-meta]] [clojure.java.io :only [input-stream]]) - (:require io.core - [schematic.core :as schema] + (:require flatland.io.core + [flatland.schematic.core :as schema] [gloss.core :as gloss])) (declare protobuf-codec) diff --git a/src/protobuf/core.clj b/src/flatland/protobuf/core.clj similarity index 90% rename from src/protobuf/core.clj rename to src/flatland/protobuf/core.clj index e5a6203..abdc0af 100644 --- a/src/protobuf/core.clj +++ b/src/flatland/protobuf/core.clj @@ -1,9 +1,9 @@ -(ns protobuf.core - (:use [protobuf.schema :only [field-schema]] - [useful.fn :only [fix]] +(ns flatland.protobuf.core + (:use [flatland.protobuf.schema :only [field-schema]] + [flatland.useful.fn :only [fix]] [clojure.java.io :only [input-stream output-stream]]) - (:require useful.utils) - (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def PersistentProtocolBufferMap$Def$NamingStrategy Extensions) + (:require flatland.useful.utils) + (:import (flatland.protobuf PersistentProtocolBufferMap PersistentProtocolBufferMap$Def PersistentProtocolBufferMap$Def$NamingStrategy Extensions) (com.google.protobuf GeneratedMessage CodedInputStream Descriptors$Descriptor) (java.io InputStream OutputStream) (clojure.lang Reflector))) @@ -69,7 +69,7 @@ (PersistentProtocolBufferMap/parseFrom type in)))) (defn ^"[B" protobuf-dump - "Return the byte representation of the given protobuf." + "Return the byte representation of the given flatland.protobuf." ([^PersistentProtocolBufferMap p] (.toByteArray p)) ([^PersistentProtocolBufferMap$Def type m] @@ -94,7 +94,7 @@ (.writeDelimitedTo p out)) (.flush out)))) -(extend-protocol useful.utils/Adjoin +(extend-protocol flatland.useful.utils/Adjoin PersistentProtocolBufferMap (adjoin-onto [^PersistentProtocolBufferMap this other] (.append this other))) diff --git a/src/protobuf/schema.clj b/src/flatland/protobuf/schema.clj similarity index 94% rename from src/protobuf/schema.clj rename to src/flatland/protobuf/schema.clj index 1702c9c..5daca83 100644 --- a/src/protobuf/schema.clj +++ b/src/flatland/protobuf/schema.clj @@ -1,10 +1,12 @@ -(ns protobuf.schema - (:use [useful.fn :only [fix]] +(ns flatland.protobuf.schema + (:use [flatland.useful.fn :only [fix]] [clojure.string :only [lower-case]]) - (:import (protobuf.core PersistentProtocolBufferMap PersistentProtocolBufferMap$Def Extensions) + (:import (flatland.protobuf PersistentProtocolBufferMap + PersistentProtocolBufferMap$Def Extensions) (com.google.protobuf Descriptors$Descriptor Descriptors$FieldDescriptor Descriptors$FieldDescriptor$Type))) + (defn extension [ext ^Descriptors$FieldDescriptor field] (-> (.getOptions field) (.getExtension ext) diff --git a/test/protobuf/codec_test.clj b/test/flatland/protobuf/codec_test.clj similarity index 89% rename from test/protobuf/codec_test.clj rename to test/flatland/protobuf/codec_test.clj index 4c062c9..5152e6e 100644 --- a/test/protobuf/codec_test.clj +++ b/test/flatland/protobuf/codec_test.clj @@ -1,9 +1,9 @@ -(ns protobuf.codec-test - (:use clojure.test gloss.io protobuf.codec) +(ns flatland.protobuf.codec-test + (:use clojure.test gloss.io flatland.protobuf.codec) (:import (java.nio ByteBuffer))) (deftest protobuf-codec-test - (let [codec (protobuf-codec protobuf.test.Codec$Foo)] + (let [codec (protobuf-codec flatland.protobuf.test.Codec$Foo)] (testing "decode an encoded data structure" (let [val {:foo 1 :bar 2}] (is (= val (decode codec (encode codec val)))))) @@ -47,8 +47,8 @@ (decode codec (concat data1 data2 data3)))))))) (deftest repeated-protobufs - (let [len (length-prefix protobuf.test.Codec$Foo) - codec (protobuf-codec protobuf.test.Codec$Foo :repeated true)] + (let [len (length-prefix flatland.protobuf.test.Codec$Foo) + codec (protobuf-codec flatland.protobuf.test.Codec$Foo :repeated true)] (testing "length-prefix" (doseq [i [0 10 100 1000 10000 100000 Integer/MAX_VALUE]] (is (= i (decode len (encode len i)))))) diff --git a/test/protobuf/core_test.clj b/test/flatland/protobuf/core_test.clj similarity index 91% rename from test/protobuf/core_test.clj rename to test/flatland/protobuf/core_test.clj index d896c33..4ecbda7 100644 --- a/test/protobuf/core_test.clj +++ b/test/flatland/protobuf/core_test.clj @@ -1,16 +1,16 @@ -(ns protobuf.core-test - (:use protobuf.core clojure.test - [io.core :only [catbytes]] - [useful.utils :only [adjoin]] +(ns flatland.protobuf.core-test + (:use flatland.protobuf.core clojure.test + [flatland.io.core :only [catbytes]] + [flatland.useful.utils :only [adjoin]] ordered-map.core) (:import (java.io PipedInputStream PipedOutputStream))) -(def Foo (protodef protobuf.test.Core$Foo)) -(def FooUnder (protodef protobuf.test.Core$Foo - {:naming-strategy protobuf.core.PersistentProtocolBufferMap$Def/protobufNames})) -(def Bar (protodef protobuf.test.Core$Bar)) -(def Response (protodef protobuf.test.Core$Response)) -(def ErrorMsg (protodef protobuf.test.Core$ErrorMsg)) +(def Foo (protodef flatland.protobuf.test.Core$Foo)) +(def FooUnder (protodef flatland.protobuf.test.Core$Foo + {:naming-strategy flatland.protobuf.PersistentProtocolBufferMap$Def/protobufNames})) +(def Bar (protodef flatland.protobuf.test.Core$Bar)) +(def Response (protodef flatland.protobuf.test.Core$Response)) +(def ErrorMsg (protodef flatland.protobuf.test.Core$ErrorMsg)) (deftest test-conj (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] @@ -263,12 +263,12 @@ (deftest test-protobuf-schema (let [fields {:type :struct - :name "protobuf.test.core.Foo" + :name "flatland.protobuf.test.core.Foo" :fields {:id {:default 43, :type :int} :deleted {:default false, :type :boolean} :lat {:type :double} :long {:type :float} - :parent {:type :struct, :name "protobuf.test.core.Foo"} + :parent {:type :struct, :name "flatland.protobuf.test.core.Foo"} :floats {:type :list, :values {:type :float}} :doubles {:type :list, :values {:type :double}} :label {:type :string, :c 3, :b 2, :a 1} @@ -276,28 +276,28 @@ :tag-set {:type :set, :values {:type :string}} :counts {:type :map :keys {:type :string} - :values {:type :struct, :name "protobuf.test.core.Count" + :values {:type :struct, :name "flatland.protobuf.test.core.Count" :fields {:key {:type :string} :i {:counter true, :type :int} :d {:counter true, :type :double}}}} :foo-by-id {:type :map :keys {:default 43, :type :int} - :values {:type :struct, :name "protobuf.test.core.Foo"}} + :values {:type :struct, :name "flatland.protobuf.test.core.Foo"}} :attr-map {:type :map :keys {:type :string} :values {:type :string}} :pair-map {:type :map :keys {:type :string} - :values {:type :struct, :name "protobuf.test.core.Pair" + :values {:type :struct, :name "flatland.protobuf.test.core.Pair" :fields {:key {:type :string} :val {:type :string}}}} :groups {:type :map :keys {:type :string} :values {:type :list - :values {:type :struct, :name "protobuf.test.core.Foo"}}} + :values {:type :struct, :name "flatland.protobuf.test.core.Foo"}}} :responses {:type :list :values {:type :enum, :values #{:no :yes :maybe :not-sure}}} - :time {:type :struct, :name "protobuf.test.core.Time", :succession true + :time {:type :struct, :name "flatland.protobuf.test.core.Time", :succession true :fields {:year {:type :int} :month {:type :int} :day {:type :int} @@ -305,11 +305,11 @@ :minute {:type :int}}} :item-map {:type :map :keys {:type :string} - :values {:type :struct, :name "protobuf.test.core.Item" + :values {:type :struct, :name "flatland.protobuf.test.core.Item" :fields {:item {:type :string}, :exists {:default true, :type :boolean}}}}}}] (is (= fields (protobuf-schema Foo))) - (is (= fields (protobuf-schema protobuf.test.Core$Foo))))) + (is (= fields (protobuf-schema flatland.protobuf.test.Core$Foo))))) (comment deftest test-default-protobuf (is (= 43 (default-protobuf Foo :id))) @@ -324,7 +324,7 @@ (is (= {} (default-protobuf Foo :groups))) (is (= {} (default-protobuf Foo :item-map))) (is (= false (default-protobuf Foo :deleted))) - (is (= {} (default-protobuf protobuf.test.Core$Foo :groups)))) + (is (= {} (default-protobuf flatland.protobuf.test.Core$Foo :groups)))) (deftest test-use-underscores (let [dashes (protobuf Foo {:tag_set ["odd"] @@ -363,7 +363,7 @@ (protobuf-seq Foo in))))) (deftest test-encoding-errors - (is (thrown-with-msg? IllegalArgumentException #"error setting string field protobuf.test.core.Foo.label to 8" + (is (thrown-with-msg? IllegalArgumentException #"error setting string field flatland.protobuf.test.core.Foo.label to 8" (protobuf Foo :label 8))) - (is (thrown-with-msg? IllegalArgumentException #"error adding 1 to string field protobuf.test.core.Foo.tags" + (is (thrown-with-msg? IllegalArgumentException #"error adding 1 to string field flatland.protobuf.test.core.Foo.tags" (protobuf Foo :tags [1 2 3])))) \ No newline at end of file diff --git a/test/protobuf/example_test.clj b/test/flatland/protobuf/example_test.clj similarity index 70% rename from test/protobuf/example_test.clj rename to test/flatland/protobuf/example_test.clj index 46aa5fb..c72b4fb 100644 --- a/test/protobuf/example_test.clj +++ b/test/flatland/protobuf/example_test.clj @@ -1,7 +1,7 @@ -(ns protobuf.example-test - (:use protobuf.core clojure.test)) +(ns flatland.protobuf.example-test + (:use flatland.protobuf.core clojure.test)) -(def Photo (protodef protobuf.test.Example$Photo)) +(def Photo (protodef flatland.protobuf.test.Example$Photo)) (def data {:id 7, :path "/photos/h2k3j4h9h23", :labels #{"hawaii" "family" "surfing"}, :attrs {"color space" "RGB", "dimensions" "1632x1224", "alpha" "no"}, From a57d4786c105f50d47b3ec32e8957f587b9b1950 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Fri, 16 Nov 2012 16:36:10 -0800 Subject: [PATCH 306/525] 0.7.0 --- project.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 20f79d9..875edb3 100644 --- a/project.clj +++ b/project.clj @@ -1,19 +1,19 @@ -(defproject org.flatland/protobuf "0.6.3-SNAPSHOT" +(defproject org.flatland/protobuf "0.7.0" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/flatland/clojure-protobuf" :dependencies [[org.clojure/clojure "1.4.0"] - [org.flatland/useful "0.8.9-SNAPSHOT"] - [org.flatland/schematic "0.1.0-SNAPSHOT"] + [org.flatland/useful "0.9.0"] + [org.flatland/schematic "0.1.0"] [ordered-collections "0.4.0"]] - :plugins [[lein-protobuf "0.2.3-SNAPSHOT"]] + :plugins [[lein-protobuf "0.3.0"]] :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} :dev {:dependencies [[gloss "0.2.1"] - [org.flatland/io "0.2.2-SNAPSHOT"]]}} + [org.flatland/io "0.3.0"]]}} :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} From e759f3fed74c2ea10c2c92a59f87d66e347fd056 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Tue, 20 Nov 2012 07:34:17 -0800 Subject: [PATCH 307/525] update to lein-protobuf 0.3.1 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 875edb3..4a358bf 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.7.0" +(defproject org.flatland/protobuf "0.7.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ [org.flatland/useful "0.9.0"] [org.flatland/schematic "0.1.0"] [ordered-collections "0.4.0"]] - :plugins [[lein-protobuf "0.3.0"]] + :plugins [[lein-protobuf "0.3.1"]] :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} From c5131eaf723d7d5bdd5effbbcb2125d371a3fefe Mon Sep 17 00:00:00 2001 From: Anthony Grimes Date: Mon, 26 Nov 2012 13:22:29 -0600 Subject: [PATCH 308/525] Update README for new nses. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14a903c..9abf799 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You'll probably want to use [Leiningen](https://github.com/technomancy/leiningen [lein-protobuf](https://github.com/flatland/lein-protobuf) plugin for compiling `.proto` files. Add the following to your `project.clj` file: - :dependencies [[protobuf "0.6.0"]] + :dependencies [[org.flatland/protobuf "0.7.1"]] :plugins [[lein-protobuf "0.1.1"]] Be sure to replace `"0.6.0"` and `"0.1.1"` with the latest versions listed at @@ -67,7 +67,7 @@ repeated fields. You can also provide metadata on protobuf fields using clojure use these, you must import the extension file and include it when compiling. For example: ```proto -import "protobuf/core/extensions.proto"; +import "flatland/protobuf/core/extensions.proto"; message Photo { required int32 id = 1; @@ -98,7 +98,7 @@ message Photo { Then you can access the extension fields in Clojure: ```clojure -(use 'protobuf) +(use 'flatland.protobuf.core) (import Example$Photo) (import Example$Photo$Tag) From 21004637624ee8dfb96b5ef17154d05b995a39bb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 3 Dec 2012 16:41:00 -0800 Subject: [PATCH 309/525] 0.7.2-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4a358bf..28cffb3 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.7.1" +(defproject org.flatland/protobuf "0.7.2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From c7f966b38fc5d6fa6e47b624a5530b87239bd182 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 3 Dec 2012 16:31:06 -0800 Subject: [PATCH 310/525] make intern handle nulls so we don't have to do it everywhere intern is called --- .../protobuf/PersistentProtocolBufferMap.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index b296af0..7352ac9 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -221,6 +221,7 @@ public Descriptors.Descriptor getMessageType() { } static final ConcurrentHashMap> caches = new ConcurrentHashMap>(); + static final Object nullv = new Object(); public Object intern(String name) { ConcurrentHashMap nameCache = caches.get(namingStrategy); @@ -233,24 +234,28 @@ public Object intern(String name) { } Object clojureName = nameCache.get(name); if (clojureName == null) { - clojureName = namingStrategy.clojureName(name); + if (name == "") { + clojureName = nullv; + } else { + clojureName = namingStrategy.clojureName(name); + if (clojureName == null) { + clojureName = nullv; + } + } Object existing = nameCache.putIfAbsent(name, clojureName); if (existing != null) { clojureName = existing; } } - return clojureName; + return clojureName == nullv ? null : clojureName; } public Object clojureEnumValue(Descriptors.EnumValueDescriptor enum_value) { return intern(enum_value.getName()); } - static final Object k_null = Keyword.intern(Symbol.intern("")); - protected Object mapFieldBy(Descriptors.FieldDescriptor field) { - Object key = intern(field.getOptions().getExtension(Extensions.mapBy)); - return key == k_null ? null : key; + return intern(field.getOptions().getExtension(Extensions.mapBy)); } } From cb3066897fccab3b461a745fbf907afa59c32cd5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 3 Dec 2012 16:34:05 -0800 Subject: [PATCH 311/525] spacing and indentation --- .../proto/flatland/protobuf/test/maps.proto | 29 +++++++++++++++++++ .../protobuf/PersistentProtocolBufferMap.java | 16 +++++----- 2 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 resources/proto/flatland/protobuf/test/maps.proto diff --git a/resources/proto/flatland/protobuf/test/maps.proto b/resources/proto/flatland/protobuf/test/maps.proto new file mode 100644 index 0000000..42891fa --- /dev/null +++ b/resources/proto/flatland/protobuf/test/maps.proto @@ -0,0 +1,29 @@ +package flatland.protobuf.test.maps; + +import "flatland/protobuf/extensions.proto"; + +option java_package = "flatland.protobuf.test"; +option java_outer_classname = "Maps"; + +message Element { + optional string id = 1; + optional int32 foo = 2; + optional int32 bar = 3; + optional bool deleted = 4; + optional bool exists = 5; +} + +message Pair { + required string key = 1; + required Element val = 2; +} + +message Struct { + repeated Pair element_map = 1 [(map) = true]; + repeated Pair element_map_e = 2 [(map) = true, (map_exists) = "exists"]; + repeated Pair element_map_d = 3 [(map) = true, (map_deleted) = "deleted"]; + + repeated Element element_by_id = 4 [(map_by) = "id"]; + repeated Element element_by_id_e = 5 [(map_by) = "id", (map_exists) = "exists"]; + repeated Element element_by_id_d = 6 [(map_by) = "id", (map_deleted) = "deleted"]; +} diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index 7352ac9..66a8191 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -370,7 +370,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value) static Keyword k_exists = Keyword.intern("exists"); protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, - boolean use_extensions) { + boolean use_extensions) { if (value instanceof List) { List values = (List)value; Iterator iterator = values.iterator(); @@ -381,8 +381,8 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, if (map_field_by != null) { ITransientMap map = (ITransientMap)OrderedMap.EMPTY.asTransient(); while (iterator.hasNext()) { - PersistentProtocolBufferMap v = (PersistentProtocolBufferMap)fromProtoValue(field, - iterator.next()); + PersistentProtocolBufferMap v = + (PersistentProtocolBufferMap)fromProtoValue(field, iterator.next()); Object k = v.valAt(map_field_by); PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap)map.valAt(k); if (existing != null) { @@ -451,8 +451,9 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, switch (field.getJavaType()) { case ENUM: Descriptors.EnumValueDescriptor e = (Descriptors.EnumValueDescriptor)value; - if (use_extensions && field.getOptions().getExtension(Extensions.nullable) - && field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { + if (use_extensions && + field.getOptions().getExtension(Extensions.nullable) && + field.getOptions().getExtension(nullExtension(field)).equals(e.getNumber())) { return null; } else { return def.clojureEnumValue(e); @@ -470,8 +471,9 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, return new PersistentProtocolBufferMap(null, fieldDef, message); default: - if (use_extensions && field.getOptions().getExtension(Extensions.nullable) - && field.getOptions().getExtension(nullExtension(field)).equals(value)) { + if (use_extensions && + field.getOptions().getExtension(Extensions.nullable) && + field.getOptions().getExtension(nullExtension(field)).equals(value)) { return null; } else { return value; From 0678d7e5a0790fa91240cd9861f8d410992871e8 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 3 Dec 2012 16:36:12 -0800 Subject: [PATCH 312/525] add map_exists and map_deleted extensions for resetting map values --- .../proto/flatland/protobuf/extensions.proto | 2 + .../protobuf/PersistentProtocolBufferMap.java | 47 +++++++-- test/flatland/protobuf/core_test.clj | 98 ++++++++++++++++++- 3 files changed, 136 insertions(+), 11 deletions(-) diff --git a/resources/proto/flatland/protobuf/extensions.proto b/resources/proto/flatland/protobuf/extensions.proto index 8fb857e..ee15932 100644 --- a/resources/proto/flatland/protobuf/extensions.proto +++ b/resources/proto/flatland/protobuf/extensions.proto @@ -9,6 +9,8 @@ extend google.protobuf.FieldOptions { optional string map_by = 52003; optional bool counter = 52004; optional bool succession = 52005; + optional string map_deleted = 52006; + optional string map_exists = 52007; optional string meta = 52010; optional bool nullable = 52020; optional string null_string = 52021; diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index 66a8191..0ca41c3 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -257,6 +257,35 @@ public Object clojureEnumValue(Descriptors.EnumValueDescriptor enum_value) { protected Object mapFieldBy(Descriptors.FieldDescriptor field) { return intern(field.getOptions().getExtension(Extensions.mapBy)); } + + protected PersistentProtocolBufferMap mapValue(Descriptors.FieldDescriptor field, + PersistentProtocolBufferMap left, + PersistentProtocolBufferMap right) { + if (left == null) { + return right; + } else { + Object map_exists = intern(field.getOptions().getExtension(Extensions.mapExists)); + if (map_exists != null) { + if (left.valAt(map_exists) == Boolean.FALSE && + right.valAt(map_exists) == Boolean.TRUE) { + return right; + } else { + return left.append(right); + } + } + + Object map_deleted = intern(field.getOptions().getExtension(Extensions.mapDeleted)); + if (map_deleted != null) { + if (left.valAt(map_deleted) == Boolean.TRUE && + right.valAt(map_deleted) == Boolean.FALSE) { + return right; + } else { + return left.append(right); + } + } + return right.append(left); + } + } } public final Def def; @@ -385,11 +414,7 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, (PersistentProtocolBufferMap)fromProtoValue(field, iterator.next()); Object k = v.valAt(map_field_by); PersistentProtocolBufferMap existing = (PersistentProtocolBufferMap)map.valAt(k); - if (existing != null) { - map = map.assoc(k, existing.append(v)); - } else { - map = map.assoc(k, v); - } + map = map.assoc(k, def.mapValue(field, existing, v)); } return map.persistent(); } else if (options.getExtension(Extensions.counter)) { @@ -411,7 +436,12 @@ protected Object fromProtoValue(Descriptors.FieldDescriptor field, Object value, Object k = fromProtoValue(key_field, message.getField(key_field)); Object v = fromProtoValue(val_field, message.getField(val_field)); Object existing = map.valAt(k); - if (existing instanceof IPersistentCollection) { + + if (existing instanceof PersistentProtocolBufferMap) { + map = map.assoc(k, def.mapValue(field, + (PersistentProtocolBufferMap)existing, + (PersistentProtocolBufferMap)v)); + } else if (existing instanceof IPersistentCollection) { map = map.assoc(k, ((IPersistentCollection)existing).cons(v)); } else { map = map.assoc(k, v); @@ -702,6 +732,11 @@ public Object getValAt(Object key, Object notFound, boolean use_extensions) { } } + protected boolean getBoolean (Object key) { + Object val = valAt(key); + return val != null && val != Boolean.FALSE; + } + @Override public PersistentProtocolBufferMap assoc(Object key, Object value) { DynamicMessage.Builder builder = builder(); diff --git a/test/flatland/protobuf/core_test.clj b/test/flatland/protobuf/core_test.clj index 4ecbda7..546d418 100644 --- a/test/flatland/protobuf/core_test.clj +++ b/test/flatland/protobuf/core_test.clj @@ -11,6 +11,7 @@ (def Bar (protodef flatland.protobuf.test.Core$Bar)) (def Response (protodef flatland.protobuf.test.Core$Response)) (def ErrorMsg (protodef flatland.protobuf.test.Core$ErrorMsg)) +(def Maps (protodef flatland.protobuf.test.Maps$Struct)) (deftest test-conj (let [p (protobuf Foo :id 5 :tags ["little" "yellow"] :doubles [1.2 3.4 5.6] :floats [0.01 0.02 0.03])] @@ -141,19 +142,106 @@ (is (= ["check" "it" "out"] (p "tags")))))) (deftest test-append-bytes - (let [p (protobuf Foo :id 5 :label "rad" :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) - q (protobuf Foo :id 43 :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) + (let [p (protobuf Foo :id 5 :label "rad" :deleted true + :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) + q (protobuf Foo :id 43 :deleted false + :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) r (protobuf Foo :label "bad") s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] - (is (= 43 (s :id))) ; make sure an explicit default overwrites on append - (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append + (is (= 43 (s :id))) ; make sure an explicit default overwrites on append + (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append (is (= "rad" (s :label))) (is (= "bad" (t :label))) (is (= ["sweet"] (t :tags))) (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) - (is (= #{"bap" "baz"} (s :tag-set))))) + (is (= #{"bap" "baz"} (s :tag-set))) + (is (= false (r :deleted))))) + +(deftest test-manual-append + (let [p (protobuf Foo :id 5 :label "rad" :deleted true + :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) + q (protobuf Foo :id 43 :deleted false + :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) + r (protobuf Foo :label "bad") + s (.append p q) + t (.append p r)] + (is (= 43 (s :id))) ; make sure an explicit default overwrites on append + (is (= 5 (t :id))) ; make sure a missing default doesn't overwrite on append + (is (= "rad" (s :label))) + (is (= "bad" (t :label))) + (is (= ["sweet"] (t :tags))) + (is (= ["sweet" "savory"] (s :tags))) + (is (= #{"foo" "bar" "baz"} (p :tag-set))) + (is (= #{"bap" "baz"} (s :tag-set))) + (is (= false (r :deleted))))) + +(deftest test-map-exists + (doseq [map-key [:element-map-e :element-by-id-e]] + (let [p (protobuf Maps map-key {"A" {:foo 1} + "B" {:foo 2} + "C" {:foo 3} + "D" {:foo 4 :exists true} + "E" {:foo 5 :exists true} + "F" {:foo 6 :exists true} + "G" {:foo 7 :exists false} + "H" {:foo 8 :exists false} + "I" {:foo 9 :exists false}}) + q (protobuf Maps map-key {"A" {:bar 1} + "B" {:bar 2 :exists true} + "C" {:bar 3 :exists false} + "D" {:bar 4} + "E" {:bar 5 :exists true} + "F" {:bar 6 :exists false} + "G" {:bar 7} + "H" {:bar 8 :exists true} + "I" {:bar 9 :exists false}}) + r (protobuf-load Maps (catbytes (protobuf-dump p) (protobuf-dump q)))] + (are [key vals] (= vals (map (get-in r [map-key key]) + [:foo :bar :exists])) + "A" [1 1 nil ] + "B" [2 2 true ] + "C" [3 3 false] + "D" [4 4 true ] + "E" [5 5 true ] + "F" [6 6 false] + "G" [7 7 false] + "H" [nil 8 true ] + "I" [9 9 false])))) + +(deftest test-map-deleted + (doseq [map-key [:element-map-d :element-by-id-d]] + (let [p (protobuf Maps map-key {"A" {:foo 1} + "B" {:foo 2} + "C" {:foo 3} + "D" {:foo 4 :deleted true} + "E" {:foo 5 :deleted true} + "F" {:foo 6 :deleted true} + "G" {:foo 7 :deleted false} + "H" {:foo 8 :deleted false} + "I" {:foo 9 :deleted false}}) + q (protobuf Maps map-key {"A" {:bar 1} + "B" {:bar 2 :deleted true} + "C" {:bar 3 :deleted false} + "D" {:bar 4} + "E" {:bar 5 :deleted true} + "F" {:bar 6 :deleted false} + "G" {:bar 7} + "H" {:bar 8 :deleted true} + "I" {:bar 9 :deleted false}}) + r (protobuf-load Maps (catbytes (protobuf-dump p) (protobuf-dump q)))] + (are [key vals] (= vals (map (get-in r [map-key key]) + [:foo :bar :deleted])) + "A" [1 1 nil ] + "B" [2 2 true ] + "C" [3 3 false] + "D" [4 4 true ] + "E" [5 5 true ] + "F" [nil 6 false] + "G" [7 7 false] + "H" [8 8 true ] + "I" [9 9 false])))) (deftest test-coercing (let [p (protobuf Foo :lat 5 :long 6)] From 8e79fb872241f67ba4bf37d77cbf58f69fa260da Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 31 Dec 2012 15:12:36 -0800 Subject: [PATCH 313/525] Add a test for append/overwrite. The test fails due to some recent changes I don't understand. @ninjudd, can you have a look at this? It fails on your map-deleted snapshot, but works on the previous release (0.7.1). --- .../proto/flatland/protobuf/test/core.proto | 8 +++++++- test/flatland/protobuf/core_test.clj | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/resources/proto/flatland/protobuf/test/core.proto b/resources/proto/flatland/protobuf/test/core.proto index 4ef5398..71b6148 100644 --- a/resources/proto/flatland/protobuf/test/core.proto +++ b/resources/proto/flatland/protobuf/test/core.proto @@ -36,6 +36,7 @@ message Foo { repeated Item item_map = 24 [(map_by) = "item"]; repeated Pair pair_map = 25 [(map_by) = "key"]; optional bool deleted = 26 [default = false]; + repeated Deletable deletions = 27 [(map_by) = "id"]; } message Bar { @@ -94,4 +95,9 @@ message ErrorMsg { message Response { required bool ok = 1; optional ErrorMsg error = 2; -} \ No newline at end of file +} + +message Deletable { + optional string id = 1; + optional bool deleted = 2; +} diff --git a/test/flatland/protobuf/core_test.clj b/test/flatland/protobuf/core_test.clj index 546d418..4f22950 100644 --- a/test/flatland/protobuf/core_test.clj +++ b/test/flatland/protobuf/core_test.clj @@ -143,9 +143,11 @@ (deftest test-append-bytes (let [p (protobuf Foo :id 5 :label "rad" :deleted true - :tags ["sweet"] :tag-set #{"foo" "bar" "baz"}) + :tags ["sweet"] :tag-set #{"foo" "bar" "baz"} + :deletions {"first" {:deleted false} "second" {:deleted false}}) q (protobuf Foo :id 43 :deleted false - :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true}) + :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true} + :deletions {"first" {:deleted true}}) r (protobuf Foo :label "bad") s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] @@ -157,6 +159,8 @@ (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) (is (= #{"bap" "baz"} (s :tag-set))) + (is (= (s :deletions) {"first" {:id "first" :deleted true} + "second" {:id "second" :deleted false}})) (is (= false (r :deleted))))) (deftest test-manual-append @@ -395,7 +399,12 @@ :keys {:type :string} :values {:type :struct, :name "flatland.protobuf.test.core.Item" :fields {:item {:type :string}, - :exists {:default true, :type :boolean}}}}}}] + :exists {:default true, :type :boolean}}}} + :deletions {:type :map + :keys {:type :string} + :values {:type :struct, :name "flatland.protobuf.test.core.Deletable" + :fields {:id {:type :string} + :deleted {:type :boolean}}}}}}] (is (= fields (protobuf-schema Foo))) (is (= fields (protobuf-schema flatland.protobuf.test.Core$Foo))))) @@ -426,7 +435,7 @@ (is (= [:yes :not_sure :maybe :not_sure :no] (:responses underscores))) (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id - :pair_map :groups :doubles :floats :item_map :counts :time :lat :long} + :pair_map :groups :doubles :floats :item_map :counts :time :lat :long :deletions} (-> (protobuf-schema FooUnder) :fields keys set))))) (deftest test-protobuf-nested-message From 58d8896fbd45eee075f65fdff436a228ffa4c336 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Mon, 31 Dec 2012 15:23:15 -0800 Subject: [PATCH 314/525] 0.7.2 has been released - update snapshot number to 0.7.3-SNAPSHOT --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 28cffb3..d706079 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.7.2-SNAPSHOT" +(defproject org.flatland/protobuf "0.7.3-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From d3b943e3fb067d4921a4d700ad02dde6e2f14188 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 2 Jan 2013 13:38:31 -0800 Subject: [PATCH 315/525] make test not use deleted so the problem is more clear --- .../proto/flatland/protobuf/test/core.proto | 6 +++--- test/flatland/protobuf/core_test.clj | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/resources/proto/flatland/protobuf/test/core.proto b/resources/proto/flatland/protobuf/test/core.proto index 71b6148..0bda650 100644 --- a/resources/proto/flatland/protobuf/test/core.proto +++ b/resources/proto/flatland/protobuf/test/core.proto @@ -36,7 +36,7 @@ message Foo { repeated Item item_map = 24 [(map_by) = "item"]; repeated Pair pair_map = 25 [(map_by) = "key"]; optional bool deleted = 26 [default = false]; - repeated Deletable deletions = 27 [(map_by) = "id"]; + repeated Thing things = 27 [(map_by) = "id"]; } message Bar { @@ -97,7 +97,7 @@ message Response { optional ErrorMsg error = 2; } -message Deletable { +message Thing { optional string id = 1; - optional bool deleted = 2; + optional bool marked = 2; } diff --git a/test/flatland/protobuf/core_test.clj b/test/flatland/protobuf/core_test.clj index 4f22950..a45b0b6 100644 --- a/test/flatland/protobuf/core_test.clj +++ b/test/flatland/protobuf/core_test.clj @@ -144,10 +144,10 @@ (deftest test-append-bytes (let [p (protobuf Foo :id 5 :label "rad" :deleted true :tags ["sweet"] :tag-set #{"foo" "bar" "baz"} - :deletions {"first" {:deleted false} "second" {:deleted false}}) + :things {"first" {:marked false} "second" {:marked false}}) q (protobuf Foo :id 43 :deleted false :tags ["savory"] :tag-set {"bar" false "foo" false "bap" true} - :deletions {"first" {:deleted true}}) + :things {"first" {:marked true}}) r (protobuf Foo :label "bad") s (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump q))) t (protobuf-load Foo (catbytes (protobuf-dump p) (protobuf-dump r)))] @@ -159,8 +159,8 @@ (is (= ["sweet" "savory"] (s :tags))) (is (= #{"foo" "bar" "baz"} (p :tag-set))) (is (= #{"bap" "baz"} (s :tag-set))) - (is (= (s :deletions) {"first" {:id "first" :deleted true} - "second" {:id "second" :deleted false}})) + (is (= (s :things) {"first" {:id "first" :marked true} + "second" {:id "second" :marked false}})) (is (= false (r :deleted))))) (deftest test-manual-append @@ -400,11 +400,11 @@ :values {:type :struct, :name "flatland.protobuf.test.core.Item" :fields {:item {:type :string}, :exists {:default true, :type :boolean}}}} - :deletions {:type :map - :keys {:type :string} - :values {:type :struct, :name "flatland.protobuf.test.core.Deletable" - :fields {:id {:type :string} - :deleted {:type :boolean}}}}}}] + :things {:type :map + :keys {:type :string} + :values {:type :struct, :name "flatland.protobuf.test.core.Thing" + :fields {:id {:type :string} + :marked {:type :boolean}}}}}}] (is (= fields (protobuf-schema Foo))) (is (= fields (protobuf-schema flatland.protobuf.test.Core$Foo))))) @@ -435,7 +435,7 @@ (is (= [:yes :not_sure :maybe :not_sure :no] (:responses underscores))) (is (= #{:id :label :tags :parent :responses :tag_set :deleted :attr_map :foo_by_id - :pair_map :groups :doubles :floats :item_map :counts :time :lat :long :deletions} + :pair_map :groups :doubles :floats :item_map :counts :time :lat :long :things} (-> (protobuf-schema FooUnder) :fields keys set))))) (deftest test-protobuf-nested-message From f0838a1ca8c4db1906b5a097708761f7082211c5 Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 2 Jan 2013 13:38:52 -0800 Subject: [PATCH 316/525] left comes before right --- src/flatland/protobuf/PersistentProtocolBufferMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index 0ca41c3..640466e 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -283,7 +283,7 @@ protected PersistentProtocolBufferMap mapValue(Descriptors.FieldDescriptor field return left.append(right); } } - return right.append(left); + return left.append(right); } } } From b14e64fda7abd9d9996de8c36f62d78d94843aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iivari=20=C3=84ik=C3=A4s?= Date: Fri, 8 Feb 2013 23:35:55 +0200 Subject: [PATCH 317/525] Mark flatland/io as project dependency --- project.clj | 4 ++-- src/flatland/protobuf/codec.clj | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index d706079..c27e07c 100644 --- a/project.clj +++ b/project.clj @@ -6,14 +6,14 @@ :dependencies [[org.clojure/clojure "1.4.0"] [org.flatland/useful "0.9.0"] [org.flatland/schematic "0.1.0"] + [org.flatland/io "0.3.0"] [ordered-collections "0.4.0"]] :plugins [[lein-protobuf "0.3.1"]] :javac-options ["-target" "5" "-source" "5"] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} - :dev {:dependencies [[gloss "0.2.1"] - [org.flatland/io "0.3.0"]]}} + :dev {:dependencies [[gloss "0.2.1"]]}} :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" :snapshots true :releases {:checksum :fail :update :always}}} diff --git a/src/flatland/protobuf/codec.clj b/src/flatland/protobuf/codec.clj index 01baecc..af4f434 100644 --- a/src/flatland/protobuf/codec.clj +++ b/src/flatland/protobuf/codec.clj @@ -5,6 +5,9 @@ [flatland.useful.fn :only [fix]] [flatland.useful.experimental :only [lift-meta]] [clojure.java.io :only [input-stream]]) + + ;; flatland.io extends Seqable so we can concat InputStream from + ;; ByteBuffer sequences. (:require flatland.io.core [flatland.schematic.core :as schema] [gloss.core :as gloss])) From 52787eb0fc5cea9949cd8a9319892a3e66e463a6 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Thu, 25 Apr 2013 10:31:49 -0700 Subject: [PATCH 318/525] Protobuf takes a map, not a naming strategy. As pointed out in #45. --- src/flatland/protobuf/core.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flatland/protobuf/core.clj b/src/flatland/protobuf/core.clj index abdc0af..1a0431d 100644 --- a/src/flatland/protobuf/core.clj +++ b/src/flatland/protobuf/core.clj @@ -23,7 +23,7 @@ ([def] (if (or (protodef? def) (nil? def)) def - (protodef def PersistentProtocolBufferMap$Def/convertUnderscores))) + (protodef def {}))) ([def opts] (when def (let [{:keys [^PersistentProtocolBufferMap$Def$NamingStrategy naming-strategy From 9f53d71f9abbc89f9b6d0c86e3bf7126dc67b295 Mon Sep 17 00:00:00 2001 From: Joe Doliner Date: Thu, 6 Jun 2013 00:25:15 -0600 Subject: [PATCH 319/525] Update the first example. The previous version would lead to the class not being found which had me (a very inexperienced clojure user) confused for a while. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9abf799..cf593b3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You can run the following to compile the `.proto` file: Now you can use the protocol buffer in Clojure: ```clojure -(use 'protobuf.core) +(use 'flatland.protobuf.core) (import Example$Person) (def Person (protodef Example$Person)) From d178d1b849b72b08a00eb88aff29814392c2c03b Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Mon, 19 Aug 2013 17:29:07 -0700 Subject: [PATCH 320/525] remove unused getBoolean method --- src/flatland/protobuf/PersistentProtocolBufferMap.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index 640466e..a722d3e 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -732,11 +732,6 @@ public Object getValAt(Object key, Object notFound, boolean use_extensions) { } } - protected boolean getBoolean (Object key) { - Object val = valAt(key); - return val != null && val != Boolean.FALSE; - } - @Override public PersistentProtocolBufferMap assoc(Object key, Object value) { DynamicMessage.Builder builder = builder(); From 5c98d41953ed21b90fcdb872041ee6287c3c0178 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 30 Oct 2013 12:42:48 -0700 Subject: [PATCH 321/525] Add test for bytestrings --- resources/proto/flatland/protobuf/test/example.proto | 1 + test/flatland/protobuf/example_test.clj | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/resources/proto/flatland/protobuf/test/example.proto b/resources/proto/flatland/protobuf/test/example.proto index fb8067b..1f8ac8d 100644 --- a/resources/proto/flatland/protobuf/test/example.proto +++ b/resources/proto/flatland/protobuf/test/example.proto @@ -11,6 +11,7 @@ message Photo { repeated Label labels = 3 [(set) = true]; repeated Attr attrs = 4 [(map) = true]; repeated Tag tags = 5 [(map_by) = "person_id"]; + optional bytes image = 6; message Label { required string item = 1; diff --git a/test/flatland/protobuf/example_test.clj b/test/flatland/protobuf/example_test.clj index c72b4fb..edee65a 100644 --- a/test/flatland/protobuf/example_test.clj +++ b/test/flatland/protobuf/example_test.clj @@ -1,11 +1,13 @@ (ns flatland.protobuf.example-test - (:use flatland.protobuf.core clojure.test)) + (:use flatland.protobuf.core clojure.test) + (:import com.google.protobuf.ByteString)) (def Photo (protodef flatland.protobuf.test.Example$Photo)) -(def data {:id 7, :path "/photos/h2k3j4h9h23", :labels #{"hawaii" "family" "surfing"}, - :attrs {"color space" "RGB", "dimensions" "1632x1224", "alpha" "no"}, - :tags {4 {:person-id 4, :x-coord 607, :y-coord 813, :width 25, :height 27}}}) +(def data {:id 7, :path "/photos/h2k3j4h9h23", :labels #{"hawaii" "family" "surfing"}, + :attrs {"color space" "RGB", "dimensions" "1632x1224", "alpha" "no"}, + :tags {4 {:person-id 4, :x-coord 607, :y-coord 813, :width 25, :height 27}} + :image (ByteString/copyFrom (byte-array (map unchecked-byte [1 2 3 4 -1])))}) (deftest example-test (is (= data (apply protobuf Photo (apply concat data))))) From d448ea547987a8d9f17779dc04e4d8d986f0adcb Mon Sep 17 00:00:00 2001 From: Justin Balthrop Date: Wed, 30 Oct 2013 11:59:13 -0700 Subject: [PATCH 322/525] 0.8.0 - update to google protobuf 2.5.0 --- project.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/project.clj b/project.clj index c27e07c..dd77cc2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.7.3-SNAPSHOT" +(defproject org.flatland/protobuf "0.8.0" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -8,8 +8,7 @@ [org.flatland/schematic "0.1.0"] [org.flatland/io "0.3.0"] [ordered-collections "0.4.0"]] - :plugins [[lein-protobuf "0.3.1"]] - :javac-options ["-target" "5" "-source" "5"] + :plugins [[lein-protobuf "0.4.1"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} From 1fee22b50719e521abd1e0e841bc364ddacb64d8 Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 30 Oct 2013 13:12:30 -0700 Subject: [PATCH 323/525] Make message() and builder() public. These were made public in the 0.7.2 jar, but due to some kind of mix-up I never committed that change so the 0.8.0 jar undoes that. --- src/flatland/protobuf/PersistentProtocolBufferMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flatland/protobuf/PersistentProtocolBufferMap.java b/src/flatland/protobuf/PersistentProtocolBufferMap.java index a722d3e..052b1f9 100644 --- a/src/flatland/protobuf/PersistentProtocolBufferMap.java +++ b/src/flatland/protobuf/PersistentProtocolBufferMap.java @@ -373,7 +373,7 @@ public Descriptors.Descriptor getMessageType() { return def.getMessageType(); } - protected DynamicMessage message() { + public DynamicMessage message() { if (message == null) { return def.newBuilder().build(); // This will only work if an empty message is valid. } else { @@ -381,7 +381,7 @@ protected DynamicMessage message() { } } - protected DynamicMessage.Builder builder() { + public DynamicMessage.Builder builder() { if (message == null) { return def.newBuilder(); } else { From 721b8f7022f5254746d2b0a3658583f7aa0f6e7c Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 30 Oct 2013 13:14:20 -0700 Subject: [PATCH 324/525] 0.8.1 - make message() and builder() public --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index dd77cc2..fb514c4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.8.0" +(defproject org.flatland/protobuf "0.8.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From ffc0f216e4d00ff730459165f5bac4e74c88f16a Mon Sep 17 00:00:00 2001 From: Alan Malloy Date: Wed, 30 Oct 2013 13:16:22 -0700 Subject: [PATCH 325/525] New snapshot --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index fb514c4..02ff5c2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.flatland/protobuf "0.8.1" +(defproject org.flatland/protobuf "0.8.2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 49896b04a8bda603b497aebfafb075e9aa131606 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 15:49:39 -0500 Subject: [PATCH 326/525] Upgrade dependencies to the latest Except gloss, it seems to crash if we upgrade Signed-off-by: Gregory Haskins --- project.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project.clj b/project.clj index 02ff5c2..1fcd22d 100644 --- a/project.clj +++ b/project.clj @@ -3,11 +3,11 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/flatland/clojure-protobuf" - :dependencies [[org.clojure/clojure "1.4.0"] - [org.flatland/useful "0.9.0"] - [org.flatland/schematic "0.1.0"] + :dependencies [[org.clojure/clojure "1.8.0"] + [org.flatland/useful "0.11.5"] + [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] - [ordered-collections "0.4.0"]] + [ordered-collections "0.4.2"]] :plugins [[lein-protobuf "0.4.1"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} From 4a5b95d54630cede07e658baf0d476a7aab618f3 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 16:07:16 -0500 Subject: [PATCH 327/525] Convert to private release in prep for deployment to clojars Signed-off-by: Gregory Haskins --- project.clj | 15 +- src/flatland/protobuf/Extensions.java | 255 ++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 10 deletions(-) create mode 100644 src/flatland/protobuf/Extensions.java diff --git a/project.clj b/project.clj index 1fcd22d..c04944d 100644 --- a/project.clj +++ b/project.clj @@ -1,20 +1,15 @@ -(defproject org.flatland/protobuf "0.8.2-SNAPSHOT" +(defproject ghaskins/clojure-protobuf "0.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :url "https://github.com/flatland/clojure-protobuf" + :url "https://github.com/ghaskins/clojure-protobuf" :dependencies [[org.clojure/clojure "1.8.0"] + [com.google.protobuf/protobuf-java "2.6.1"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] - [ordered-collections "0.4.2"]] - :plugins [[lein-protobuf "0.4.1"]] + [ordered-collections "0.4.2"] + [gloss "0.2.1"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} - :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} - :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} - :dev {:dependencies [[gloss "0.2.1"]]}} - :repositories {"sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" - :snapshots true - :releases {:checksum :fail :update :always}}} :checksum-deps true :java-source-paths ["src"]) diff --git a/src/flatland/protobuf/Extensions.java b/src/flatland/protobuf/Extensions.java new file mode 100644 index 0000000..c141191 --- /dev/null +++ b/src/flatland/protobuf/Extensions.java @@ -0,0 +1,255 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: flatland/protobuf/extensions.proto + +package flatland.protobuf; + +public final class Extensions { + private Extensions() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registry.add(flatland.protobuf.Extensions.set); + registry.add(flatland.protobuf.Extensions.map); + registry.add(flatland.protobuf.Extensions.mapBy); + registry.add(flatland.protobuf.Extensions.counter); + registry.add(flatland.protobuf.Extensions.succession); + registry.add(flatland.protobuf.Extensions.mapDeleted); + registry.add(flatland.protobuf.Extensions.mapExists); + registry.add(flatland.protobuf.Extensions.meta); + registry.add(flatland.protobuf.Extensions.nullable); + registry.add(flatland.protobuf.Extensions.nullString); + registry.add(flatland.protobuf.Extensions.nullInt); + registry.add(flatland.protobuf.Extensions.nullLong); + registry.add(flatland.protobuf.Extensions.nullFloat); + registry.add(flatland.protobuf.Extensions.nullDouble); + registry.add(flatland.protobuf.Extensions.nullEnum); + } + public static final int SET_FIELD_NUMBER = 52001; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Boolean> set = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Boolean.class, + null); + public static final int MAP_FIELD_NUMBER = 52002; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Boolean> map = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Boolean.class, + null); + public static final int MAP_BY_FIELD_NUMBER = 52003; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.String> mapBy = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.String.class, + null); + public static final int COUNTER_FIELD_NUMBER = 52004; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Boolean> counter = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Boolean.class, + null); + public static final int SUCCESSION_FIELD_NUMBER = 52005; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Boolean> succession = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Boolean.class, + null); + public static final int MAP_DELETED_FIELD_NUMBER = 52006; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.String> mapDeleted = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.String.class, + null); + public static final int MAP_EXISTS_FIELD_NUMBER = 52007; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.String> mapExists = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.String.class, + null); + public static final int META_FIELD_NUMBER = 52010; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.String> meta = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.String.class, + null); + public static final int NULLABLE_FIELD_NUMBER = 52020; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Boolean> nullable = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Boolean.class, + null); + public static final int NULL_STRING_FIELD_NUMBER = 52021; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.String> nullString = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.String.class, + null); + public static final int NULL_INT_FIELD_NUMBER = 52022; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Integer> nullInt = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Integer.class, + null); + public static final int NULL_LONG_FIELD_NUMBER = 52023; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Long> nullLong = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Long.class, + null); + public static final int NULL_FLOAT_FIELD_NUMBER = 52024; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Float> nullFloat = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Float.class, + null); + public static final int NULL_DOUBLE_FIELD_NUMBER = 52025; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Double> nullDouble = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Double.class, + null); + public static final int NULL_ENUM_FIELD_NUMBER = 52026; + /** + * extend .google.protobuf.FieldOptions { ... } + */ + public static final + com.google.protobuf.GeneratedMessage.GeneratedExtension< + com.google.protobuf.DescriptorProtos.FieldOptions, + java.lang.Integer> nullEnum = com.google.protobuf.GeneratedMessage + .newFileScopedGeneratedExtension( + java.lang.Integer.class, + null); + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\"flatland/protobuf/extensions.proto\032 go" + + "ogle/protobuf/descriptor.proto:,\n\003set\022\035." + + "google.protobuf.FieldOptions\030\241\226\003 \001(\010:,\n\003" + + "map\022\035.google.protobuf.FieldOptions\030\242\226\003 \001" + + "(\010:/\n\006map_by\022\035.google.protobuf.FieldOpti" + + "ons\030\243\226\003 \001(\t:0\n\007counter\022\035.google.protobuf" + + ".FieldOptions\030\244\226\003 \001(\010:3\n\nsuccession\022\035.go" + + "ogle.protobuf.FieldOptions\030\245\226\003 \001(\010:4\n\013ma" + + "p_deleted\022\035.google.protobuf.FieldOptions" + + "\030\246\226\003 \001(\t:3\n\nmap_exists\022\035.google.protobuf", + ".FieldOptions\030\247\226\003 \001(\t:-\n\004meta\022\035.google.p" + + "rotobuf.FieldOptions\030\252\226\003 \001(\t:1\n\010nullable" + + "\022\035.google.protobuf.FieldOptions\030\264\226\003 \001(\010:" + + "4\n\013null_string\022\035.google.protobuf.FieldOp" + + "tions\030\265\226\003 \001(\t:1\n\010null_int\022\035.google.proto" + + "buf.FieldOptions\030\266\226\003 \001(\021:2\n\tnull_long\022\035." + + "google.protobuf.FieldOptions\030\267\226\003 \001(\022:3\n\n" + + "null_float\022\035.google.protobuf.FieldOption" + + "s\030\270\226\003 \001(\002:4\n\013null_double\022\035.google.protob" + + "uf.FieldOptions\030\271\226\003 \001(\001:2\n\tnull_enum\022\035.g", + "oogle.protobuf.FieldOptions\030\272\226\003 \001(\rB\037\n\021f" + + "latland.protobufB\nExtensions" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + com.google.protobuf.DescriptorProtos.getDescriptor(), + }, assigner); + set.internalInit(descriptor.getExtensions().get(0)); + map.internalInit(descriptor.getExtensions().get(1)); + mapBy.internalInit(descriptor.getExtensions().get(2)); + counter.internalInit(descriptor.getExtensions().get(3)); + succession.internalInit(descriptor.getExtensions().get(4)); + mapDeleted.internalInit(descriptor.getExtensions().get(5)); + mapExists.internalInit(descriptor.getExtensions().get(6)); + meta.internalInit(descriptor.getExtensions().get(7)); + nullable.internalInit(descriptor.getExtensions().get(8)); + nullString.internalInit(descriptor.getExtensions().get(9)); + nullInt.internalInit(descriptor.getExtensions().get(10)); + nullLong.internalInit(descriptor.getExtensions().get(11)); + nullFloat.internalInit(descriptor.getExtensions().get(12)); + nullDouble.internalInit(descriptor.getExtensions().get(13)); + nullEnum.internalInit(descriptor.getExtensions().get(14)); + com.google.protobuf.DescriptorProtos.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} From 7c38d8393be6fa4f49b765af8167bae0cb8290eb Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 16:17:48 -0500 Subject: [PATCH 328/525] Update project name Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c04944d..33f7d9d 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject ghaskins/clojure-protobuf "0.1-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "0.1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 81662bf034face2815f2e18b74df0a1ceba8928e Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 16:41:51 -0500 Subject: [PATCH 329/525] Manage the minimum java compatibility to remain compatible with OBC JRE Signed-off-by: Gregory Haskins --- project.clj | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 33f7d9d..b5316cb 100644 --- a/project.clj +++ b/project.clj @@ -3,6 +3,8 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/ghaskins/clojure-protobuf" + :javac-options ["-target" "1.7" "-source" "1.7"] + :java-source-paths ["src"] :dependencies [[org.clojure/clojure "1.8.0"] [com.google.protobuf/protobuf-java "2.6.1"] [org.flatland/useful "0.11.5"] @@ -11,5 +13,4 @@ [ordered-collections "0.4.2"] [gloss "0.2.1"]] :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} - :checksum-deps true - :java-source-paths ["src"]) + :checksum-deps true) From f9099c4431aaa74bcf95fbf2bf307bb157fbce13 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 16:22:20 -0500 Subject: [PATCH 330/525] Update README.md --- README.md | 84 ++++--------------------------------------------------- 1 file changed, 5 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index cf593b3..e150c95 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,11 @@ clojure-protobuf provides a Clojure interface to Google's [protocol buffers](http://code.google.com/p/protobuf). -Protocol buffers can be used to communicate with other languages over the network, and -they are WAY faster to serialize and deserialize than standard Clojure objects. +Protocol buffers can be used to communicate with other languages over the network, and they are WAY faster to serialize and deserialize than standard Clojure objects. ## Getting started -You'll probably want to use [Leiningen](https://github.com/technomancy/leiningen) with the -[lein-protobuf](https://github.com/flatland/lein-protobuf) plugin for compiling `.proto` files. Add -the following to your `project.clj` file: +Add the dependency to your project.clj - :dependencies [[org.flatland/protobuf "0.7.1"]] - :plugins [[lein-protobuf "0.1.1"]] - -Be sure to replace `"0.6.0"` and `"0.1.1"` with the latest versions listed at -http://clojars.org/protobuf and http://clojars.org/lein-protobuf. - -*Note: lein-protobuf requires at least version 2.0 of Leiningen.* - -## Usage +[![Clojars Project](https://img.shields.io/clojars/v/org.clojars.ghaskins/protobuf.svg)](https://clojars.org/org.clojars.ghaskins/protobuf) Assuming you have the following in `resources/proto/person.proto`: @@ -29,9 +18,9 @@ message Person { } ``` -You can run the following to compile the `.proto` file: +Compile the proto using the protobuf compiler and include the resulting .java code in your project - lein protobuf + protoc --java_out=./ -proto_dir=resources/proto person.proto Now you can use the protocol buffer in Clojure: @@ -60,66 +49,3 @@ Now you can use the protocol buffer in Clojure: A protocol buffer map is immutable just like other clojure objects. It is similar to a struct-map, except you cannot insert fields that aren't specified in the `.proto` file. -## Extensions - -Clojure-protobuf supports extensions to protocol buffers which provide sets and maps using -repeated fields. You can also provide metadata on protobuf fields using clojure syntax. To -use these, you must import the extension file and include it when compiling. For example: - -```proto -import "flatland/protobuf/core/extensions.proto"; - -message Photo { - required int32 id = 1; - required string path = 2; - repeated Label labels = 3 [(set) = true]; - repeated Attr attrs = 4 [(map) = true]; - repeated Tag tags = 5 [(map_by) = "person_id"]; - - message Label { - required string item = 1; - required bool exists = 2; - } - - message Attr { - required string key = 1; - optional string val = 2; - } - - message Tag { - required int32 person_id = 1; - optional int32 x_coord = 2 [(meta) = "{:max 100.0 :min -100.0}"]; - optional int32 y_coord = 3; - optional int32 width = 4; - optional int32 height = 5; - } -} -``` -Then you can access the extension fields in Clojure: - -```clojure -(use 'flatland.protobuf.core) -(import Example$Photo) -(import Example$Photo$Tag) - -(def Photo (protodef Example$Photo)) -(def Tag (protodef Example$Photo$Tag)) - -(def p (protobuf Photo :id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"} - :attrs {"dimensions" "1632x1224", "alpha" "no", "color space" "RGB"} - :tags {4 {:person_id 4, :x_coord 607, :y_coord 813, :width 25, :height 27}})) -=> {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} - -(def b (protobuf-dump p)) -=> # - -(protobuf-load Photo b) -=> {:id 7 :path "/photos/h2k3j4h9h23" :labels #{"hawaii" "family" "surfing"}...} - -(:x-coord (protobuf-schema Tag)) -=> {:max 100.0 :min -100.0} -``` - -## Getting Help - -If you have any questions or need help, you can find us on IRC in [#flatland](irc://irc.freenode.net/#flatland). From a36f305467d472658b93b7cef447b06809191bd8 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 16:44:20 -0500 Subject: [PATCH 331/525] Add lein-release support for deploying to clojars Signed-off-by: Gregory Haskins --- project.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/project.clj b/project.clj index b5316cb..c2db072 100644 --- a/project.clj +++ b/project.clj @@ -5,6 +5,7 @@ :url "https://github.com/ghaskins/clojure-protobuf" :javac-options ["-target" "1.7" "-source" "1.7"] :java-source-paths ["src"] + :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] [com.google.protobuf/protobuf-java "2.6.1"] [org.flatland/useful "0.11.5"] From 8fdef4e1a3e731f49324d72e621c57a9e565339c Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 17:05:22 -0500 Subject: [PATCH 332/525] Add repository info Signed-off-by: Gregory Haskins --- project.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project.clj b/project.clj index c2db072..f4da670 100644 --- a/project.clj +++ b/project.clj @@ -6,6 +6,8 @@ :javac-options ["-target" "1.7" "-source" "1.7"] :java-source-paths ["src"] :lein-release {:deploy-via :clojars} + :repositories [["releases" {:url "https://clojars.org/repo" + :creds :gpg}]] :dependencies [[org.clojure/clojure "1.8.0"] [com.google.protobuf/protobuf-java "2.6.1"] [org.flatland/useful "0.11.5"] From 625499c48348b61c6e24e12772f8ef4548845752 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 17:23:21 -0500 Subject: [PATCH 333/525] More clojars tweaking Signed-off-by: Gregory Haskins --- project.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/project.clj b/project.clj index f4da670..c2db072 100644 --- a/project.clj +++ b/project.clj @@ -6,8 +6,6 @@ :javac-options ["-target" "1.7" "-source" "1.7"] :java-source-paths ["src"] :lein-release {:deploy-via :clojars} - :repositories [["releases" {:url "https://clojars.org/repo" - :creds :gpg}]] :dependencies [[org.clojure/clojure "1.8.0"] [com.google.protobuf/protobuf-java "2.6.1"] [org.flatland/useful "0.11.5"] From c09854d98bbc06b4b5789dd2b414b9567119a3a5 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 17:23:39 -0500 Subject: [PATCH 334/525] lein-release plugin: preparing 0.1 release --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c2db072..e4650af 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.1-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "0.1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 16c8e983a23fe10e5db77229baee3a9245cc69c7 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 18 Feb 2016 17:24:13 -0500 Subject: [PATCH 335/525] lein-release plugin: bumped version from 0.1 to 0.2-SNAPSHOT for next development cycle --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index e4650af..07fecc0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.1" +(defproject org.clojars.ghaskins/protobuf "0.2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 27207c8ffbc29ca3517224664466582002b8a85f Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 17 May 2016 14:53:59 -0400 Subject: [PATCH 336/525] Upgrade to proto3 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 07fecc0..e4ee572 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "2.6.1"] + [com.google.protobuf/protobuf-java "3.0.0-beta-2"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From 798b1b00fb0bd2643d74305da0da22ff797e89b4 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 17 May 2016 14:57:34 -0400 Subject: [PATCH 337/525] lein-release plugin: preparing 0.2 release --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index e4ee572..ee38a07 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.2-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "0.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 94e2c72846c11e1f0acfd743f8499c4ffa9b1c71 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 17 May 2016 14:58:21 -0400 Subject: [PATCH 338/525] lein-release plugin: bumped version from 0.2 to 0.3-SNAPSHOT for next development cycle --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index ee38a07..9f19b33 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.2" +(defproject org.clojars.ghaskins/protobuf "0.3-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 28fe07099347ebc8ab6ffdc6be601ef000b2624b Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Wed, 29 Jun 2016 08:48:10 -0400 Subject: [PATCH 339/525] Update to protobuf-3-beta3 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 9f19b33..9fd724e 100644 --- a/project.clj +++ b/project.clj @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.0.0-beta-2"] + [com.google.protobuf/protobuf-java "3.0.0-beta-3"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From 01367bb62865ca40eb4a43fad9edc8a6c5321dd6 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Wed, 29 Jun 2016 09:38:16 -0400 Subject: [PATCH 340/525] Version v0.3 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 9fd724e..2aea4e8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.3-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "0.3" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 411353c6ca2860fa866aa8869bf1cadff483acdf Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Wed, 29 Jun 2016 09:38:55 -0400 Subject: [PATCH 341/525] Prepare for v0.4 development Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 2aea4e8..0eeabb6 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.3" +(defproject org.clojars.ghaskins/protobuf "0.4-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 2278a863131bf6269dc9b51075123d7f6a997780 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 22 Sep 2016 11:45:14 -0400 Subject: [PATCH 342/525] Upgrade to v3.0.2 released protobuf We also change our versioning scheme to reflect the protobuf library we are using. The schema is $PROTOVER-$RELEASE, e.g. 3.0.2-1 Signed-off-by: Gregory Haskins --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 0eeabb6..77709ad 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "0.4-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.0.2-1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.0.0-beta-3"] + [com.google.protobuf/protobuf-java "3.0.2"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From 73b066fa646020cff518afcf6cc336a7668df3f7 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 22 Sep 2016 12:08:54 -0400 Subject: [PATCH 343/525] Version 3.0.2-1 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 77709ad..706475e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.0.2-1-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.0.2-1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 350a622a23992cbbcd319e6edfa9848fba56541d Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 22 Sep 2016 13:20:52 -0400 Subject: [PATCH 344/525] Prepare for v3.0.2-2 development Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 706475e..70e7f1c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.0.2-1" +(defproject org.clojars.ghaskins/protobuf "3.0.2-2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 1d5ff2002270d3aee431bd147a1dbfb2bb2a7c2d Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 22 Sep 2016 14:45:15 -0400 Subject: [PATCH 345/525] Upgrade to java 1.8 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 70e7f1c..4776ec1 100644 --- a/project.clj +++ b/project.clj @@ -3,7 +3,7 @@ :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/ghaskins/clojure-protobuf" - :javac-options ["-target" "1.7" "-source" "1.7"] + :javac-options ["-target" "1.8" "-source" "1.8"] :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] From 2ca00a2814f2df0d7face2a0e75b4eb9e8cfeaa8 Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Thu, 22 Sep 2016 14:52:06 -0400 Subject: [PATCH 346/525] Release v3.0.2-2 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4776ec1..1d449e4 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.0.2-2-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.0.2-2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From b826ed571b0655d0dab4c65724e63c20876f3816 Mon Sep 17 00:00:00 2001 From: Kiran Karkera Date: Tue, 16 May 2017 15:37:06 +0800 Subject: [PATCH 347/525] moved to protobuf version 3.1.0 --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 1d449e4..4884ba1 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.0.2-2" +(defproject org.clojars.ghaskins/protobuf "3.1.0" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.0.2"] + [com.google.protobuf/protobuf-java "3.1.0"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From a4618c69e2b829ca5efc8300341fd47192d64f4c Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 23 May 2017 14:31:29 -0400 Subject: [PATCH 348/525] Release v3.1.0-1 Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 4884ba1..1364258 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.1.0" +(defproject org.clojars.ghaskins/protobuf "3.1.0-1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 2d2e1371ba126257d2210a81f7e609dea884ca1a Mon Sep 17 00:00:00 2001 From: Gregory Haskins Date: Tue, 23 May 2017 14:32:15 -0400 Subject: [PATCH 349/525] Prepare for v3.1.0-2 development Signed-off-by: Gregory Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 1364258..168b5f1 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.1.0-1" +(defproject org.clojars.ghaskins/protobuf "3.1.0-2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 51db19398344aea8883499626fa9dd28c9c6bc7b Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Tue, 23 May 2017 14:45:54 -0400 Subject: [PATCH 350/525] Upgrade to protobuf v3.3.1 Signed-off-by: Greg Haskins --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 168b5f1..095bfc0 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.1.0-2-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.3.1-1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.1.0"] + [com.google.protobuf/protobuf-java "3.3.1"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From 81e9935cedfd67c59d85e8611b76996d0f45906c Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Tue, 23 May 2017 14:46:33 -0400 Subject: [PATCH 351/525] Release v3.3.1-1 Signed-off-by: Greg Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 095bfc0..fe7ead8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.3.1-1-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.3.1-1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 941118f26362db9128b072054b220992cb7d52e0 Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Tue, 23 May 2017 14:47:06 -0400 Subject: [PATCH 352/525] Prepare for v3.3.1-2 development Signed-off-by: Greg Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index fe7ead8..baec365 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.3.1-1" +(defproject org.clojars.ghaskins/protobuf "3.3.1-2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 786c991c88ea64d8d36231339bac3decd7340700 Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Fri, 1 Sep 2017 17:31:35 -0400 Subject: [PATCH 353/525] Upgrade to protobuf v3.4.0 Signed-off-by: Greg Haskins --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index baec365..676439a 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.3.1-2-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.4.0-1-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} @@ -7,7 +7,7 @@ :java-source-paths ["src"] :lein-release {:deploy-via :clojars} :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.3.1"] + [com.google.protobuf/protobuf-java "3.4.0"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] From b2fd75bd99c878b630f778e95021e05141aa030c Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Fri, 1 Sep 2017 17:32:02 -0400 Subject: [PATCH 354/525] Release v3.4.0-1 Signed-off-by: Greg Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 676439a..54fe6c8 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.4.0-1-SNAPSHOT" +(defproject org.clojars.ghaskins/protobuf "3.4.0-1" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 2c873d4bb0825138b9506a3d4524be19a9f22050 Mon Sep 17 00:00:00 2001 From: Greg Haskins Date: Fri, 1 Sep 2017 17:32:29 -0400 Subject: [PATCH 355/525] Prepare for v3.4.0-2 development Signed-off-by: Greg Haskins --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 54fe6c8..cc1d4d6 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.clojars.ghaskins/protobuf "3.4.0-1" +(defproject org.clojars.ghaskins/protobuf "3.4.0-2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From 095a42d3e336d55f50b58c870527ddf014a5e7aa Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 10:29:54 -0600 Subject: [PATCH 356/525] Create a transitional release for the clojusc org. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index cc1d4d6..53afa4c 100644 --- a/project.clj +++ b/project.clj @@ -1,8 +1,8 @@ -(defproject org.clojars.ghaskins/protobuf "3.4.0-2-SNAPSHOT" +(defproject clojusc/protobuf "3.4.0-2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :url "https://github.com/ghaskins/clojure-protobuf" + :url "https://github.com/clojusc/clojure-protobuf" :javac-options ["-target" "1.8" "-source" "1.8"] :java-source-paths ["src"] :lein-release {:deploy-via :clojars} From 64737adff88a20489e90f294ee1281593dc5c4f7 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 11:18:33 -0600 Subject: [PATCH 357/525] Establish a convention for releases; functionally identical to ghaskins' b9c386e. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 53afa4c..c32cc0e 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojusc/protobuf "3.4.0-2-SNAPSHOT" +(defproject clojusc/protobuf "3.4.0-v0.2-SNAPSHOT" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From df20f2b862d542498ec531d688dbef2ddd896778 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 11:20:50 -0600 Subject: [PATCH 358/525] Release as-is. --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index c32cc0e..1fd0b3c 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject clojusc/protobuf "3.4.0-v0.2-SNAPSHOT" +(defproject clojusc/protobuf "3.4.0-v0.2" :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} From edb6dc47b9a95074e3f8608eefc627998f4ed6c4 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 11:29:20 -0600 Subject: [PATCH 359/525] Bump version for next dev cycle. --- project.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 1fd0b3c..b35dc2e 100644 --- a/project.clj +++ b/project.clj @@ -1,5 +1,5 @@ -(defproject clojusc/protobuf "3.4.0-v0.2" - :description "Clojure-protobuf provides a clojure interface to Google's protocol buffers." +(defproject clojusc/protobuf "3.4.0-v0.3-SNAPSHOT" + :description "A Clojure interface to Google's protocol buffers" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/clojusc/clojure-protobuf" From c8919ec96ec2a300ce1c02e4ac0786f6af5783fe Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 10:37:31 -0600 Subject: [PATCH 360/525] Updated Travis CI config. --- .travis.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 65a5762..d1e6b30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,12 @@ language: clojure -lein: lein2 -script: lein2 testall \ No newline at end of file +sudo: false +script: lein testall +jdk: + - oraclejdk8 + - oraclejdk7 + - openjdk6 + - openjdk7 +notifications: + email: + - oubiwann@gmail.com + From 125ea790f4671a493825093836ec2ecbabe3ee29 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 10:40:32 -0600 Subject: [PATCH 361/525] Re-added old testing profiles. --- project.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/project.clj b/project.clj index b35dc2e..db326c8 100644 --- a/project.clj +++ b/project.clj @@ -2,7 +2,7 @@ :description "A Clojure interface to Google's protocol buffers" :license {:name "Eclipse Public License" :url "http://www.eclipse.org/legal/epl-v10.html"} - :url "https://github.com/clojusc/clojure-protobuf" + :url "https://github.com/clojusc/protobuf" :javac-options ["-target" "1.8" "-source" "1.8"] :java-source-paths ["src"] :lein-release {:deploy-via :clojars} @@ -13,5 +13,8 @@ [org.flatland/io "0.3.0"] [ordered-collections "0.4.2"] [gloss "0.2.1"]] + :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} + :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} + :dev {:dependencies [[gloss "0.2.1"]]}} :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} :checksum-deps true) From 8bdaace452b2ca1ff37221f05c47ac1d975a093c Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 11:13:57 -0600 Subject: [PATCH 362/525] Re-add lein-protobuf. --- project.clj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project.clj b/project.clj index db326c8..1573c17 100644 --- a/project.clj +++ b/project.clj @@ -13,6 +13,8 @@ [org.flatland/io "0.3.0"] [ordered-collections "0.4.2"] [gloss "0.2.1"]] + :plugins [[lein-protobuf "0.5.0"]] + :protobuf-version "3.4.0" :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} :dev {:dependencies [[gloss "0.2.1"]]}} From f57f909be91dc9e4205cf929197b3206e9d7e381 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 12:51:09 -0600 Subject: [PATCH 363/525] Updated ignores. --- .gitignore | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index ba264f3..338a8b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,12 @@ -.lein-deps-sum -.lein-failures +.lein-* .cake *~ lib classes build -pom.xml -pom.xml.asc +pom.xml* *.jar *.class -proto/google/protobuf/descriptor.proto docs protobuf-* protosrc From 42cc8d2c425881002dade4a6614200012932fe55 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 12:55:02 -0600 Subject: [PATCH 364/525] Got tests running again. --- bin/compile-test-protobufs | 11 ++++++++ project.clj | 52 +++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 18 deletions(-) create mode 100755 bin/compile-test-protobufs diff --git a/bin/compile-test-protobufs b/bin/compile-test-protobufs new file mode 100755 index 0000000..64430c0 --- /dev/null +++ b/bin/compile-test-protobufs @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +OUT_DIR=target/test + +mkdir -p $OUT_DIR +protoc \ + -I=/usr/include \ + -I=/usr/local/include \ + -I=resources/proto \ + --java_out=$OUT_DIR \ + resources/proto/flatland/protobuf/test/*.proto diff --git a/project.clj b/project.clj index 1573c17..9e6e3a2 100644 --- a/project.clj +++ b/project.clj @@ -1,22 +1,38 @@ (defproject clojusc/protobuf "3.4.0-v0.3-SNAPSHOT" :description "A Clojure interface to Google's protocol buffers" - :license {:name "Eclipse Public License" - :url "http://www.eclipse.org/legal/epl-v10.html"} :url "https://github.com/clojusc/protobuf" - :javac-options ["-target" "1.8" "-source" "1.8"] + :license { + :name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :exclusions [ + [org.clojure/clojure]] + :dependencies [ + [org.clojure/clojure "1.8.0"] + [com.google.protobuf/protobuf-java "3.4.0"] + [org.flatland/useful "0.11.5"] + [org.flatland/schematic "0.1.5"] + [org.flatland/io "0.3.0"] + [ordered-collections "0.4.2"] + [gloss "0.2.1"]] :java-source-paths ["src"] - :lein-release {:deploy-via :clojars} - :dependencies [[org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.4.0"] - [org.flatland/useful "0.11.5"] - [org.flatland/schematic "0.1.5"] - [org.flatland/io "0.3.0"] - [ordered-collections "0.4.2"] - [gloss "0.2.1"]] - :plugins [[lein-protobuf "0.5.0"]] - :protobuf-version "3.4.0" - :profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} - :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]} - :dev {:dependencies [[gloss "0.2.1"]]}} - :aliases {"testall" ["with-profile" "dev,default:dev,1.3,default:dev,1.5,default" "test"]} - :checksum-deps true) + :profiles { + :test { + :plugins [[lein-shell "0.5.0"]] + :java-source-paths ["target/test"]} + :1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} + :1.5 {:dependencies [[org.clojure/clojure "1.5.0"]]}} + :aliases { + "protoc-test" [ + "with-profile" + "+test" + "shell" + "bin/compile-test-protobufs"] + "test-all" [ + "with-profile" + "+1.3,+1.5,+default" + "test"] + "build-test" [ + "do" + ["clean"] + ["protoc-test"] + ["test-all"]]}) From 1c51040972ee167f8f07e585952fc2d4b6e4a94d Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 13:02:08 -0600 Subject: [PATCH 365/525] Add protoc to Travis build. --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d1e6b30..afb0c93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: clojure sudo: false -script: lein testall +before_install: + - sudo apt-get -qq update + - sudo apt-get install -y protobuf-compiler libprotoc-dev +script: lein build-test jdk: - oraclejdk8 - oraclejdk7 From bd3d8705f01f314549179a9b1ffd7e30e658e402 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 15:01:04 -0600 Subject: [PATCH 366/525] Added more versions to test. --- project.clj | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/project.clj b/project.clj index 9e6e3a2..e077f4f 100644 --- a/project.clj +++ b/project.clj @@ -19,8 +19,10 @@ :test { :plugins [[lein-shell "0.5.0"]] :java-source-paths ["target/test"]} - :1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]} - :1.5 {:dependencies [[org.clojure/clojure "1.5.0"]]}} + :1.5 {:dependencies [[org.clojure/clojure "1.5.0"]]} + :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} + :1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} + :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]}} :aliases { "protoc-test" [ "with-profile" @@ -29,10 +31,10 @@ "bin/compile-test-protobufs"] "test-all" [ "with-profile" - "+1.3,+1.5,+default" + "+1.5:+1.6:+1.7:+1.9:+default" "test"] "build-test" [ "do" - ["clean"] - ["protoc-test"] - ["test-all"]]}) + ["clean"] + ["protoc-test"] + ["test-all"]]}) From bbd1e1d5b62e9cbcc1f344758ad6789dd202b4f2 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 15:19:00 -0600 Subject: [PATCH 367/525] Added linting profiles and fixed linting errors. --- project.clj | 23 ++++++++++++++++++++--- src/flatland/protobuf/codec.clj | 24 ++++++++++++------------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/project.clj b/project.clj index e077f4f..bc539c2 100644 --- a/project.clj +++ b/project.clj @@ -8,16 +8,21 @@ [org.clojure/clojure]] :dependencies [ [org.clojure/clojure "1.8.0"] - [com.google.protobuf/protobuf-java "3.4.0"] + [com.google.protobuf/protobuf-java "3.5.1"] [org.flatland/useful "0.11.5"] [org.flatland/schematic "0.1.5"] [org.flatland/io "0.3.0"] [ordered-collections "0.4.2"] - [gloss "0.2.1"]] + [gloss "0.2.6"]] :java-source-paths ["src"] :profiles { :test { - :plugins [[lein-shell "0.5.0"]] + :dependencies [ + [clojusc/ltest "0.3.0-SNAPSHOT"]] + :plugins [ + [jonase/eastwood "0.2.5"] + [lein-ancient "0.6.15"] + [lein-shell "0.5.0"]] :java-source-paths ["target/test"]} :1.5 {:dependencies [[org.clojure/clojure "1.5.0"]]} :1.6 {:dependencies [[org.clojure/clojure "1.6.0"]]} @@ -29,6 +34,17 @@ "+test" "shell" "bin/compile-test-protobufs"] + "check-deps" [ + "with-profile" + "+test" + "ancient" + "check" + ":all"] + "lint" [ + "with-profile" + "+test" + "eastwood" + "{:namespaces [:source-paths] :source-paths [\"src\"]}"] "test-all" [ "with-profile" "+1.5:+1.6:+1.7:+1.9:+default" @@ -36,5 +52,6 @@ "build-test" [ "do" ["clean"] + ["lint"] ["protoc-test"] ["test-all"]]}) diff --git a/src/flatland/protobuf/codec.clj b/src/flatland/protobuf/codec.clj index af4f434..7f9f6eb 100644 --- a/src/flatland/protobuf/codec.clj +++ b/src/flatland/protobuf/codec.clj @@ -1,16 +1,16 @@ (ns flatland.protobuf.codec - (:use flatland.protobuf.core - [gloss.core.protocols :only [Reader Writer]] - [gloss.core.formats :only [to-buf-seq]] - [flatland.useful.fn :only [fix]] - [flatland.useful.experimental :only [lift-meta]] - [clojure.java.io :only [input-stream]]) - - ;; flatland.io extends Seqable so we can concat InputStream from - ;; ByteBuffer sequences. - (:require flatland.io.core - [flatland.schematic.core :as schema] - [gloss.core :as gloss])) + (:require + [clojure.java.io :refer [input-stream]] + ;; flatland.io extends Seqable so we can concat InputStream from + ;; ByteBuffer sequences. + [flatland.io.core] + [flatland.protobuf.core :refer :all] + [flatland.schematic.core :as schema] + [flatland.useful.experimental :refer [lift-meta]] + [flatland.useful.fn :refer [fix]] + [gloss.core :as gloss] + [gloss.core.formats :refer [to-buf-seq]] + [gloss.core.protocols :refer [Reader Writer]])) (declare protobuf-codec) From 14f15b1c9c0b1d8483929e3d07ff5b0274359540 Mon Sep 17 00:00:00 2001 From: Duncan McGreggor Date: Sat, 23 Dec 2017 15:40:49 -0600 Subject: [PATCH 368/525] Added codox-generated docs. --- .gitignore | 1 - docs/current/css/default.css | 638 +++++++++++++++++++++ docs/current/flatland.protobuf.codec.html | 3 + docs/current/flatland.protobuf.core.html | 3 + docs/current/flatland.protobuf.schema.html | 3 + docs/current/highlight/atom-one-dark.css | 96 ++++ docs/current/highlight/highlight.min.js | 2 + docs/current/highlight/highlight.pack.js | 2 + docs/current/highlight/solarized-light.css | 84 +++ docs/current/index.html | 3 + docs/current/js/jquery.min.js | 4 + docs/current/js/page_effects.js | 113 ++++ docs/index.html | 2 + project.clj | 22 +- resources/docs/9999-other-versions.md | 5 + 15 files changed, 979 insertions(+), 2 deletions(-) create mode 100644 docs/current/css/default.css create mode 100644 docs/current/flatland.protobuf.codec.html create mode 100644 docs/current/flatland.protobuf.core.html create mode 100644 docs/current/flatland.protobuf.schema.html create mode 100644 docs/current/highlight/atom-one-dark.css create mode 100644 docs/current/highlight/highlight.min.js create mode 100644 docs/current/highlight/highlight.pack.js create mode 100644 docs/current/highlight/solarized-light.css create mode 100644 docs/current/index.html create mode 100644 docs/current/js/jquery.min.js create mode 100644 docs/current/js/page_effects.js create mode 100644 docs/index.html create mode 100644 resources/docs/9999-other-versions.md diff --git a/.gitignore b/.gitignore index 338a8b2..6c512bb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ build pom.xml* *.jar *.class -docs protobuf-* protosrc .classpath diff --git a/docs/current/css/default.css b/docs/current/css/default.css new file mode 100644 index 0000000..89df9b5 --- /dev/null +++ b/docs/current/css/default.css @@ -0,0 +1,638 @@ +@import url('https://fonts.googleapis.com/css?family=PT+Sans'); + +body { + font-family: 'PT Sans', Helvetica, sans-serif; + font-size: 14px; +} + +a { + color: #337ab7; + text-decoration: none; +} + +a:hover { + color: #30426a; + text-decoration: underline; +} + +pre, code { + font-family: Monaco, DejaVu Sans Mono, Consolas, monospace; + font-size: 9pt; + margin: 15px 0; +} + +h1 { + font-weight: normal; + font-size: 29px; + margin: 10px 0 2px 0; + padding: 0; +} + +h2 { + font-weight: normal; + font-size: 25px; +} + +h3 > a:hover { + text-decoration: none; +} + +.document h1, .namespace-index h1 { + font-size: 32px; + margin-top: 12px; +} + +#header, #content, .sidebar { + position: fixed; +} + +#header { + top: 0; + left: 0; + right: 0; + height: 22px; + color: #f5f5f5; + padding: 5px 7px; +} + +#content { + top: 32px; + right: 0; + bottom: 0; + overflow: auto; + background: #eee; + color: #333; + padding: 0 18px; +} + +.sidebar { + position: fixed; + top: 32px; + bottom: 0; + overflow: auto; + color: white; +} + +.sidebar.primary { + background: #1f283b; + border-right: solid 1px #1f283b; + left: 0; + width: 250px; + font-size: 110%; +} + +.sidebar.secondary { + background: #2a364f; + border-right: solid 1px #d7d7d7; + left: 251px; + width: 200px; + font-size: 110%; +} + +#content.namespace-index, #content.document { + left: 251px; +} + +#content.namespace-docs { + left: 452px; +} + +#content.document { + padding-bottom: 10%; + left: 200px; +} + +#header { + background: #58852b; + box-shadow: 0 0 8px rgba(0, 0, 0, 0.4); + z-index: 100; +} + +#header h1 { + margin: 0; + padding: 0; + font-size: 18px; + font-weight: lighter; +} + +#header h1 .project-version { + font-weight: normal; +} + +.project-version { + padding-left: 0.15em; +} + +#header a, .sidebar a { + display: block; + text-decoration: none; +} + +#header a { + color: #f5f5f5; +} + +.sidebar.primary, .sidebar.primary a, +.sidebar.secondary, .sidebar.secondary a { + color: #b2bfdc; +} + +.sidebar.primary a:hover, .sidebar.secondary a:hover { + color: white; +} + +#header h2 { + float: right; + font-size: 9pt; + font-weight: normal; + margin: 4px 3px; + padding: 0; + color: #bbb; +} + +#header h2 a { + display: inline; +} + +.sidebar h3 { + margin: 0; + padding: 10px 13px 0 13px; + font-size: 19px; + font-weight: lighter; +} + +.sidebar.primary h3.no-link, .sidebar.secondary h3 a { + text-transform: uppercase; + font-size: 12px; + color: #738bc0; +} + +.sidebar ul { + padding: 7px 0 6px 0; + margin: 0; +} + +.sidebar ul.index-link { + padding-bottom: 4px; +} + +.sidebar li { + display: block; + vertical-align: middle; +} + +.sidebar li a, .sidebar li .no-link { + border-left: 3px solid transparent; + padding: 0 10px; + white-space: nowrap; +} + +.sidebar li .inner { + display: inline-block; + padding-top: 7px; + height: 24px; +} + +.sidebar li a, .sidebar li .tree { + height: 31px; +} + +.depth-1 .inner { padding-left: 2px; } +.depth-2 .inner { padding-left: 6px; } +.depth-3 .inner { padding-left: 20px; } +.depth-4 .inner { padding-left: 34px; } +.depth-5 .inner { padding-left: 48px; } +.depth-6 .inner { padding-left: 62px; } +.depth-7 .inner { padding-left: 76px; } +.depth-8 .inner { padding-left: 90px; } + +.sidebar li .tree { + display: block; + float: left; + position: relative; + top: -10px; + margin: 0 4px 0 0; + padding: 0; +} + +.sidebar li.depth-1 .tree { + display: none; +} + +.sidebar li .tree .top, .sidebar li .tree .bottom { + display: block; + margin: 0; + padding: 0; + width: 7px; +} + +.sidebar li .tree .top { + border-left: 1px solid #aaa; + border-bottom: 1px solid #aaa; + height: 19px; +} + +.sidebar li .tree .bottom { + height: 22px; +} + +.sidebar li.branch .tree .bottom { + border-left: 1px solid #aaa; +} + +.sidebar.primary li.current a, +.sidebar.secondary li.current a { + border-left: 3px solid #58852b; + color: white; +} + +.namespace-index h2 { + margin: 30px 0 0 0; +} + +.namespace-index h3 { + font-size: 16px; + font-weight: bold; + margin-bottom: 0; + letter-spacing: 0.05em; + border-bottom: solid 1px #ddd; + max-width: 680px; + background-color: #fafafa; + padding: 0.5em; +} + +.namespace-index .topics { + padding-left: 30px; + margin: 11px 0 0 0; +} + +ul.topics { + list-style-type: none; +} + +.namespace-index .topics li { + padding: 5px 0; +} + +.namespace-docs h3 { + font-size: 18px; + font-weight: bold; +} + +.public h3 { + margin: 0; + float: left; +} + +.usage { + clear: both; +} + +.public { + margin: 0; + border-top: 1px solid #e0e0e0; + padding-top: 14px; + padding-bottom: 6px; +} + +.public:last-child { + margin-bottom: 20%; +} + +.members .public:last-child { + margin-bottom: 0; +} + +.members { + margin: 15px 0; +} + +.members h4 { + color: #555; + font-weight: normal; + font-variant: small-caps; + margin: 0 0 5px 0; +} + +.members .inner { + padding-top: 5px; + padding-left: 12px; + margin-top: 2px; + margin-left: 7px; + border-left: 1px solid #bbb; +} + +#content .members .inner h3 { + font-size: 12pt; +} + +.members .public { + border-top: none; + margin-top: 0; + padding-top: 6px; + padding-bottom: 0; +} + +.members .public:first-child { + padding-top: 0; +} + +h4.type, +h4.dynamic, +h4.added, +h4.deprecated { + float: left; + margin: 3px 10px 15px 0; + font-size: 15px; + font-weight: bold; + font-variant: small-caps; +} + +.public h4.type, +.public h4.dynamic, +.public h4.added, +.public h4.deprecated { + font-size: 13px; + font-weight: bold; + margin: 3px 0 0 10px; +} + +.members h4.type, +.members h4.added, +.members h4.deprecated { + margin-top: 1px; +} + +h4.type { + color: #717171; +} + +h4.dynamic { + color: #9933aa; +} + +h4.added { + color: #508820; +} + +h4.deprecated { + color: #880000; +} + +.namespace { + margin-bottom: 30px; +} + +.namespace:last-child { + margin-bottom: 10%; +} + +.index { + padding: 0; + font-size: 80%; + margin: 15px 0; + line-height: 1.6em; +} + +.index * { + display: inline; +} + +.index p { + padding-right: 3px; +} + +.index li { + padding-right: 5px; +} + +.index ul { + padding-left: 0; +} + +.type-sig { + clear: both; + color: #088; +} + +.type-sig pre { + padding-top: 10px; + margin: 0; +} + +.usage code { + display: block; + color: #008; + margin: 2px 0; +} + +.usage code:first-child { + padding-top: 10px; +} + +p { + margin: 15px 0; +} + +.public p:first-child, .public pre.plaintext { + margin-top: 12px; +} + +.doc { + margin: 0 0 26px 0; + clear: both; +} + +.public .doc { + margin: 0; +} + +.namespace-index { + font-size: 120%; +} + +.namespace-index .doc { + margin-bottom: 20px; +} + +.namespace-index .namespace .doc { + margin-bottom: 10px; +} + +.markdown p, .markdown li, .markdown dt, .markdown dd, .markdown td { + line-height: 1.6em; +} + +.markdown h2 { + font-weight: normal; + font-size: 25px; +} + +#content .markdown h3 { + font-size: 20px; +} + +.markdown h4 { + font-size: 15px; +} + +.doc, .public, .namespace .index { + max-width: 680px; + overflow-x: visible; +} + +.markdown pre > code { + display: block; + padding: 10px; +} + +.markdown pre > code, .src-link a { + border: 1px solid #e4e4e4; + border-radius: 2px; +} + +.src-link a { + background: #f6f6f6; +} + +.markdown code:not(.hljs) { + color: #c7254e; + background-color: #f9f2f4; + border-radius: 4px; + font-size: 90%; + padding: 2px 4px; +} + +pre.deps { + display: inline-block; + margin: 0 10px; + border: 1px solid #e4e4e4; + border-radius: 2px; + padding: 10px; + background-color: #f6f6f6; +} + +.markdown hr { + border-style: solid; + border-top: none; + color: #ccc; +} + +.doc ul, .doc ol { + padding-left: 30px; +} + +.doc table { + border-collapse: collapse; + margin: 0 10px; +} + +.doc table td, .doc table th { + border: 1px solid #dddddd; + padding: 4px 6px; +} + +.doc table th { + background: #f2f2f2; +} + +.doc dl { + margin: 0 10px 20px 10px; +} + +.doc dl dt { + font-weight: bold; + margin: 0; + padding: 3px 0; + border-bottom: 1px solid #ddd; +} + +.doc dl dd { + padding: 5px 0; + margin: 0 0 5px 10px; +} + +.doc abbr { + border-bottom: 1px dotted #333; + font-variant: none; + cursor: help; +} + +.src-link { + margin-bottom: 15px; +} + +.src-link a { + font-size: 70%; + padding: 1px 4px; + text-decoration: none; + color: #5555bb; + background-color: #f6f6f6; +} + +blockquote { + opacity: 0.6; + border-left: solid 2px #ddd; + margin-left: 0; + padding-left: 1em; +} + +/* Responsiveness Theme */ + +@media (max-device-width: 480px) { + .sidebar { + display:none; + } + + #content { + position: relative; + left: initial !important; + top: 110px; + padding: 1em 1em 0 1em; + } + + #header { + display: flex; + flex-direction: column-reverse; + height: 100px; + } + + #header > h1 { + font-size: 52px; + } + + #header h2 { + float: none; + font-size: 20px; + } + + .namespace-index > h1 { + display: none; + } + + .public, .doc, .namespace > .index, .namespace > .doc, .namespace > h3 { + max-width: initial; + } + + .doc { + text-align: justify; + } + + .public { + padding-top: 2em; + padding-bottom: 2em; + } + + .public > h3 { + font-size: 300%; + } + + .public > h4.type, .public > h4.added, .public > h4.deprecated { + font-size: 150%; + margin-top: 1em; + } + + pre > code { + font-size: 200%; + } +} diff --git a/docs/current/flatland.protobuf.codec.html b/docs/current/flatland.protobuf.codec.html new file mode 100644 index 0000000..426bbab --- /dev/null +++ b/docs/current/flatland.protobuf.codec.html @@ -0,0 +1,3 @@ + +flatland.protobuf.codec documentation

flatland.protobuf.codec

Documentation forthcoming

codec-schema

(codec-schema proto)

Documentation forthcoming

length-prefix

(length-prefix proto)

Documentation forthcoming

protobuf-codec

(protobuf-codec proto & {:keys [validator repeated]})

Documentation forthcoming

\ No newline at end of file diff --git a/docs/current/flatland.protobuf.core.html b/docs/current/flatland.protobuf.core.html new file mode 100644 index 0000000..558fda5 --- /dev/null +++ b/docs/current/flatland.protobuf.core.html @@ -0,0 +1,3 @@ + +flatland.protobuf.core documentation

flatland.protobuf.core

Documentation forthcoming

get-raw

(get-raw p key)

Get value at key ignoring extension fields.

protobuf

(protobuf type)(protobuf type m)(protobuf type k v & kvs)

Construct a protobuf of the given type.

protobuf-dump

(protobuf-dump p)(protobuf-dump type m)

Return the byte representation of the given flatland.protobuf.

protobuf-load

(protobuf-load type data)(protobuf-load type data offset length)

Load a protobuf of the given type from an array of bytes.

protobuf-load-stream

(protobuf-load-stream type stream)

Load a protobuf of the given type from an InputStream.

protobuf-schema

(protobuf-schema & args)

Return the schema for the given protodef.

protobuf-seq

(protobuf-seq type in)

Lazily read a sequence of length-delimited protobufs of the specified type from the given input stream.

protobuf-write

(protobuf-write out & ps)

Write the given protobufs to the given output stream, prefixing each with its length to delimit them.

protobuf?

(protobuf? obj)

Is the given object a PersistentProtocolBufferMap?

protodef

(protodef def)(protodef def opts)

Create a protodef from a string or protobuf class.

protodef?

(protodef? obj)

Is the given object a PersistentProtocolBufferMap$Def?

\ No newline at end of file diff --git a/docs/current/flatland.protobuf.schema.html b/docs/current/flatland.protobuf.schema.html new file mode 100644 index 0000000..fbb54d4 --- /dev/null +++ b/docs/current/flatland.protobuf.schema.html @@ -0,0 +1,3 @@ + +flatland.protobuf.schema documentation

flatland.protobuf.schema

Documentation forthcoming

basic-schema

(basic-schema field def & [parents])

Documentation forthcoming

extension

(extension ext field)

Documentation forthcoming

field-schema

multimethod

Documentation forthcoming

field-type

(field-type field)

Documentation forthcoming

struct-schema

(struct-schema struct def & [parents])

Documentation forthcoming

subfield

(subfield field field-name)

Documentation forthcoming

\ No newline at end of file diff --git a/docs/current/highlight/atom-one-dark.css b/docs/current/highlight/atom-one-dark.css new file mode 100644 index 0000000..1616aaf --- /dev/null +++ b/docs/current/highlight/atom-one-dark.css @@ -0,0 +1,96 @@ +/* + +Atom One Dark by Daniel Gamage +Original One Dark Syntax theme from https://github.com/atom/one-dark-syntax + +base: #282c34 +mono-1: #abb2bf +mono-2: #818896 +mono-3: #5c6370 +hue-1: #56b6c2 +hue-2: #61aeee +hue-3: #c678dd +hue-4: #98c379 +hue-5: #e06c75 +hue-5-2: #be5046 +hue-6: #d19a66 +hue-6-2: #e6c07b + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + color: #abb2bf; + background: #282c34; +} + +.hljs-comment, +.hljs-quote { + color: #5c6370; + font-style: italic; +} + +.hljs-doctag, +.hljs-keyword, +.hljs-formula { + color: #c678dd; +} + +.hljs-section, +.hljs-name, +.hljs-selector-tag, +.hljs-deletion, +.hljs-subst { + color: #e06c75; +} + +.hljs-literal { + color: #56b6c2; +} + +.hljs-string, +.hljs-regexp, +.hljs-addition, +.hljs-attribute, +.hljs-meta-string { + color: #98c379; +} + +.hljs-built_in, +.hljs-class .hljs-title { + color: #e6c07b; +} + +.hljs-attr, +.hljs-variable, +.hljs-template-variable, +.hljs-type, +.hljs-selector-class, +.hljs-selector-attr, +.hljs-selector-pseudo, +.hljs-number { + color: #d19a66; +} + +.hljs-symbol, +.hljs-bullet, +.hljs-link, +.hljs-meta, +.hljs-selector-id, +.hljs-title { + color: #61aeee; +} + +.hljs-emphasis { + font-style: italic; +} + +.hljs-strong { + font-weight: bold; +} + +.hljs-link { + text-decoration: underline; +} diff --git a/docs/current/highlight/highlight.min.js b/docs/current/highlight/highlight.min.js new file mode 100644 index 0000000..6486ffd --- /dev/null +++ b/docs/current/highlight/highlight.min.js @@ -0,0 +1,2 @@ +/*! highlight.js v9.6.0 | BSD3 License | git.io/hljslicense */ +!function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"!=typeof exports?e(exports):n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs}))}(function(e){function n(e){return e.replace(/[&<>]/gm,function(e){return I[e]})}function t(e){return e.nodeName.toLowerCase()}function r(e,n){var t=e&&e.exec(n);return t&&0===t.index}function a(e){return k.test(e)}function i(e){var n,t,r,i,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",t=B.exec(o))return R(t[1])?t[1]:"no-highlight";for(o=o.split(/\s+/),n=0,r=o.length;r>n;n++)if(i=o[n],a(i)||R(i))return i}function o(e,n){var t,r={};for(t in e)r[t]=e[t];if(n)for(t in n)r[t]=n[t];return r}function u(e){var n=[];return function r(e,a){for(var i=e.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=r(i,a),t(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n}function c(e,r,a){function i(){return e.length&&r.length?e[0].offset!==r[0].offset?e[0].offset"}function u(e){l+=""}function c(e){("start"===e.event?o:u)(e.node)}for(var s=0,l="",f=[];e.length||r.length;){var g=i();if(l+=n(a.substr(s,g[0].offset-s)),s=g[0].offset,g===e){f.reverse().forEach(u);do c(g.splice(0,1)[0]),g=i();while(g===e&&g.length&&g[0].offset===s);f.reverse().forEach(o)}else"start"===g[0].event?f.push(g[0].node):f.pop(),c(g.splice(0,1)[0])}return l+n(a.substr(s))}function s(e){function n(e){return e&&e.source||e}function t(t,r){return new RegExp(n(t),"m"+(e.cI?"i":"")+(r?"g":""))}function r(a,i){if(!a.compiled){if(a.compiled=!0,a.k=a.k||a.bK,a.k){var u={},c=function(n,t){e.cI&&(t=t.toLowerCase()),t.split(" ").forEach(function(e){var t=e.split("|");u[t[0]]=[n,t[1]?Number(t[1]):1]})};"string"==typeof a.k?c("keyword",a.k):E(a.k).forEach(function(e){c(e,a.k[e])}),a.k=u}a.lR=t(a.l||/\w+/,!0),i&&(a.bK&&(a.b="\\b("+a.bK.split(" ").join("|")+")\\b"),a.b||(a.b=/\B|\b/),a.bR=t(a.b),a.e||a.eW||(a.e=/\B|\b/),a.e&&(a.eR=t(a.e)),a.tE=n(a.e)||"",a.eW&&i.tE&&(a.tE+=(a.e?"|":"")+i.tE)),a.i&&(a.iR=t(a.i)),null==a.r&&(a.r=1),a.c||(a.c=[]);var s=[];a.c.forEach(function(e){e.v?e.v.forEach(function(n){s.push(o(e,n))}):s.push("self"===e?a:e)}),a.c=s,a.c.forEach(function(e){r(e,a)}),a.starts&&r(a.starts,i);var l=a.c.map(function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b}).concat([a.tE,a.i]).map(n).filter(Boolean);a.t=l.length?t(l.join("|"),!0):{exec:function(){return null}}}}r(e)}function l(e,t,a,i){function o(e,n){var t,a;for(t=0,a=n.c.length;a>t;t++)if(r(n.c[t].bR,e))return n.c[t]}function u(e,n){if(r(e.eR,n)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?u(e.parent,n):void 0}function c(e,n){return!a&&r(n.iR,e)}function g(e,n){var t=N.cI?n[0].toLowerCase():n[0];return e.k.hasOwnProperty(t)&&e.k[t]}function h(e,n,t,r){var a=r?"":y.classPrefix,i='',i+n+o}function p(){var e,t,r,a;if(!E.k)return n(B);for(a="",t=0,E.lR.lastIndex=0,r=E.lR.exec(B);r;)a+=n(B.substr(t,r.index-t)),e=g(E,r),e?(M+=e[1],a+=h(e[0],n(r[0]))):a+=n(r[0]),t=E.lR.lastIndex,r=E.lR.exec(B);return a+n(B.substr(t))}function d(){var e="string"==typeof E.sL;if(e&&!x[E.sL])return n(B);var t=e?l(E.sL,B,!0,L[E.sL]):f(B,E.sL.length?E.sL:void 0);return E.r>0&&(M+=t.r),e&&(L[E.sL]=t.top),h(t.language,t.value,!1,!0)}function b(){k+=null!=E.sL?d():p(),B=""}function v(e){k+=e.cN?h(e.cN,"",!0):"",E=Object.create(e,{parent:{value:E}})}function m(e,n){if(B+=e,null==n)return b(),0;var t=o(n,E);if(t)return t.skip?B+=n:(t.eB&&(B+=n),b(),t.rB||t.eB||(B=n)),v(t,n),t.rB?0:n.length;var r=u(E,n);if(r){var a=E;a.skip?B+=n:(a.rE||a.eE||(B+=n),b(),a.eE&&(B=n));do E.cN&&(k+=C),E.skip||(M+=E.r),E=E.parent;while(E!==r.parent);return r.starts&&v(r.starts,""),a.rE?0:n.length}if(c(n,E))throw new Error('Illegal lexeme "'+n+'" for mode "'+(E.cN||"")+'"');return B+=n,n.length||1}var N=R(e);if(!N)throw new Error('Unknown language: "'+e+'"');s(N);var w,E=i||N,L={},k="";for(w=E;w!==N;w=w.parent)w.cN&&(k=h(w.cN,"",!0)+k);var B="",M=0;try{for(var I,j,O=0;;){if(E.t.lastIndex=O,I=E.t.exec(t),!I)break;j=m(t.substr(O,I.index-O),I[0]),O=I.index+j}for(m(t.substr(O)),w=E;w.parent;w=w.parent)w.cN&&(k+=C);return{r:M,value:k,language:e,top:E}}catch(T){if(T.message&&-1!==T.message.indexOf("Illegal"))return{r:0,value:n(t)};throw T}}function f(e,t){t=t||y.languages||E(x);var r={r:0,value:n(e)},a=r;return t.filter(R).forEach(function(n){var t=l(n,e,!1);t.language=n,t.r>a.r&&(a=t),t.r>r.r&&(a=r,r=t)}),a.language&&(r.second_best=a),r}function g(e){return y.tabReplace||y.useBR?e.replace(M,function(e,n){return y.useBR&&"\n"===e?"
":y.tabReplace?n.replace(/\t/g,y.tabReplace):void 0}):e}function h(e,n,t){var r=n?L[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}function p(e){var n,t,r,o,s,p=i(e);a(p)||(y.useBR?(n=document.createElementNS("http://www.w3.org/1999/xhtml","div"),n.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")):n=e,s=n.textContent,r=p?l(p,s,!0):f(s),t=u(n),t.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=r.value,r.value=c(t,u(o),s)),r.value=g(r.value),e.innerHTML=r.value,e.className=h(e.className,p,r.language),e.result={language:r.language,re:r.r},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.r}))}function d(e){y=o(y,e)}function b(){if(!b.called){b.called=!0;var e=document.querySelectorAll("pre code");w.forEach.call(e,p)}}function v(){addEventListener("DOMContentLoaded",b,!1),addEventListener("load",b,!1)}function m(n,t){var r=x[n]=t(e);r.aliases&&r.aliases.forEach(function(e){L[e]=n})}function N(){return E(x)}function R(e){return e=(e||"").toLowerCase(),x[e]||x[L[e]]}var w=[],E=Object.keys,x={},L={},k=/^(no-?highlight|plain|text)$/i,B=/\blang(?:uage)?-([\w-]+)\b/i,M=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,C="
",y={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0},I={"&":"&","<":"<",">":">"};return e.highlight=l,e.highlightAuto=f,e.fixMarkup=g,e.highlightBlock=p,e.configure=d,e.initHighlighting=b,e.initHighlightingOnLoad=v,e.registerLanguage=m,e.listLanguages=N,e.getLanguage=R,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|like)\b/},e.C=function(n,t,r){var a=e.inherit({cN:"comment",b:n,e:t,c:[]},r||{});return a.c.push(e.PWM),a.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),a},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e});hljs.registerLanguage("clojure",function(e){var t={"builtin-name":"def defonce cond apply if-not if-let if not not= = < > <= >= == + / * - rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit defmacro defn defn- macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},r="a-zA-Z_\\-!.?+*=<>&#'",n="["+r+"]["+r+"0-9/;:]*",a="[-+]?\\d+(\\.\\d+)?",o={b:n,r:0},s={cN:"number",b:a,r:0},i=e.inherit(e.QSM,{i:null}),c=e.C(";","$",{r:0}),d={cN:"literal",b:/\b(true|false|nil)\b/},l={b:"[\\[\\{]",e:"[\\]\\}]"},m={cN:"comment",b:"\\^"+n},p=e.C("\\^\\{","\\}"),u={cN:"symbol",b:"[:]{1,2}"+n},f={b:"\\(",e:"\\)"},h={eW:!0,r:0},y={k:t,l:n,cN:"name",b:n,starts:h},b=[f,i,m,p,c,u,l,s,d,o];return f.c=[e.C("comment",""),y,h],h.c=b,l.c=b,{aliases:["clj"],i:/\S/,c:[f,i,m,p,c,u,l,s,d]}});hljs.registerLanguage("clojure-repl",function(e){return{c:[{cN:"meta",b:/^([\w.-]+|\s*#_)=>/,starts:{e:/$/,sL:"clojure"}}]}}); \ No newline at end of file diff --git a/docs/current/highlight/highlight.pack.js b/docs/current/highlight/highlight.pack.js new file mode 100644 index 0000000..126e338 --- /dev/null +++ b/docs/current/highlight/highlight.pack.js @@ -0,0 +1,2 @@ +/*! highlight.js v9.11.0 | BSD3 License | git.io/hljslicense */ +!function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"!=typeof exports?e(exports):n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs}))}(function(e){function n(e){return e.replace(/&/g,"&").replace(//g,">")}function t(e){return e.nodeName.toLowerCase()}function r(e,n){var t=e&&e.exec(n);return t&&0===t.index}function a(e){return k.test(e)}function i(e){var n,t,r,i,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",t=B.exec(o))return w(t[1])?t[1]:"no-highlight";for(o=o.split(/\s+/),n=0,r=o.length;r>n;n++)if(i=o[n],a(i)||w(i))return i}function o(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach(function(e){for(n in e)t[n]=e[n]}),t}function u(e){var n=[];return function r(e,a){for(var i=e.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=r(i,a),t(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n}function c(e,r,a){function i(){return e.length&&r.length?e[0].offset!==r[0].offset?e[0].offset"}function u(e){s+=""}function c(e){("start"===e.event?o:u)(e.node)}for(var l=0,s="",f=[];e.length||r.length;){var g=i();if(s+=n(a.substring(l,g[0].offset)),l=g[0].offset,g===e){f.reverse().forEach(u);do c(g.splice(0,1)[0]),g=i();while(g===e&&g.length&&g[0].offset===l);f.reverse().forEach(o)}else"start"===g[0].event?f.push(g[0].node):f.pop(),c(g.splice(0,1)[0])}return s+n(a.substr(l))}function l(e){return e.v&&!e.cached_variants&&(e.cached_variants=e.v.map(function(n){return o(e,{v:null},n)})),e.cached_variants||e.eW&&[o(e)]||[e]}function s(e){function n(e){return e&&e.source||e}function t(t,r){return new RegExp(n(t),"m"+(e.cI?"i":"")+(r?"g":""))}function r(a,i){if(!a.compiled){if(a.compiled=!0,a.k=a.k||a.bK,a.k){var o={},u=function(n,t){e.cI&&(t=t.toLowerCase()),t.split(" ").forEach(function(e){var t=e.split("|");o[t[0]]=[n,t[1]?Number(t[1]):1]})};"string"==typeof a.k?u("keyword",a.k):x(a.k).forEach(function(e){u(e,a.k[e])}),a.k=o}a.lR=t(a.l||/\w+/,!0),i&&(a.bK&&(a.b="\\b("+a.bK.split(" ").join("|")+")\\b"),a.b||(a.b=/\B|\b/),a.bR=t(a.b),a.e||a.eW||(a.e=/\B|\b/),a.e&&(a.eR=t(a.e)),a.tE=n(a.e)||"",a.eW&&i.tE&&(a.tE+=(a.e?"|":"")+i.tE)),a.i&&(a.iR=t(a.i)),null==a.r&&(a.r=1),a.c||(a.c=[]),a.c=Array.prototype.concat.apply([],a.c.map(function(e){return l("self"===e?a:e)})),a.c.forEach(function(e){r(e,a)}),a.starts&&r(a.starts,i);var c=a.c.map(function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b}).concat([a.tE,a.i]).map(n).filter(Boolean);a.t=c.length?t(c.join("|"),!0):{exec:function(){return null}}}}r(e)}function f(e,t,a,i){function o(e,n){var t,a;for(t=0,a=n.c.length;a>t;t++)if(r(n.c[t].bR,e))return n.c[t]}function u(e,n){if(r(e.eR,n)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?u(e.parent,n):void 0}function c(e,n){return!a&&r(n.iR,e)}function l(e,n){var t=N.cI?n[0].toLowerCase():n[0];return e.k.hasOwnProperty(t)&&e.k[t]}function p(e,n,t,r){var a=r?"":I.classPrefix,i='',i+n+o}function h(){var e,t,r,a;if(!E.k)return n(k);for(a="",t=0,E.lR.lastIndex=0,r=E.lR.exec(k);r;)a+=n(k.substring(t,r.index)),e=l(E,r),e?(B+=e[1],a+=p(e[0],n(r[0]))):a+=n(r[0]),t=E.lR.lastIndex,r=E.lR.exec(k);return a+n(k.substr(t))}function d(){var e="string"==typeof E.sL;if(e&&!y[E.sL])return n(k);var t=e?f(E.sL,k,!0,x[E.sL]):g(k,E.sL.length?E.sL:void 0);return E.r>0&&(B+=t.r),e&&(x[E.sL]=t.top),p(t.language,t.value,!1,!0)}function b(){L+=null!=E.sL?d():h(),k=""}function v(e){L+=e.cN?p(e.cN,"",!0):"",E=Object.create(e,{parent:{value:E}})}function m(e,n){if(k+=e,null==n)return b(),0;var t=o(n,E);if(t)return t.skip?k+=n:(t.eB&&(k+=n),b(),t.rB||t.eB||(k=n)),v(t,n),t.rB?0:n.length;var r=u(E,n);if(r){var a=E;a.skip?k+=n:(a.rE||a.eE||(k+=n),b(),a.eE&&(k=n));do E.cN&&(L+=C),E.skip||(B+=E.r),E=E.parent;while(E!==r.parent);return r.starts&&v(r.starts,""),a.rE?0:n.length}if(c(n,E))throw new Error('Illegal lexeme "'+n+'" for mode "'+(E.cN||"")+'"');return k+=n,n.length||1}var N=w(e);if(!N)throw new Error('Unknown language: "'+e+'"');s(N);var R,E=i||N,x={},L="";for(R=E;R!==N;R=R.parent)R.cN&&(L=p(R.cN,"",!0)+L);var k="",B=0;try{for(var M,j,O=0;;){if(E.t.lastIndex=O,M=E.t.exec(t),!M)break;j=m(t.substring(O,M.index),M[0]),O=M.index+j}for(m(t.substr(O)),R=E;R.parent;R=R.parent)R.cN&&(L+=C);return{r:B,value:L,language:e,top:E}}catch(T){if(T.message&&-1!==T.message.indexOf("Illegal"))return{r:0,value:n(t)};throw T}}function g(e,t){t=t||I.languages||x(y);var r={r:0,value:n(e)},a=r;return t.filter(w).forEach(function(n){var t=f(n,e,!1);t.language=n,t.r>a.r&&(a=t),t.r>r.r&&(a=r,r=t)}),a.language&&(r.second_best=a),r}function p(e){return I.tabReplace||I.useBR?e.replace(M,function(e,n){return I.useBR&&"\n"===e?"
":I.tabReplace?n.replace(/\t/g,I.tabReplace):""}):e}function h(e,n,t){var r=n?L[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}function d(e){var n,t,r,o,l,s=i(e);a(s)||(I.useBR?(n=document.createElementNS("http://www.w3.org/1999/xhtml","div"),n.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")):n=e,l=n.textContent,r=s?f(s,l,!0):g(l),t=u(n),t.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=r.value,r.value=c(t,u(o),l)),r.value=p(r.value),e.innerHTML=r.value,e.className=h(e.className,s,r.language),e.result={language:r.language,re:r.r},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.r}))}function b(e){I=o(I,e)}function v(){if(!v.called){v.called=!0;var e=document.querySelectorAll("pre code");E.forEach.call(e,d)}}function m(){addEventListener("DOMContentLoaded",v,!1),addEventListener("load",v,!1)}function N(n,t){var r=y[n]=t(e);r.aliases&&r.aliases.forEach(function(e){L[e]=n})}function R(){return x(y)}function w(e){return e=(e||"").toLowerCase(),y[e]||y[L[e]]}var E=[],x=Object.keys,y={},L={},k=/^(no-?highlight|plain|text)$/i,B=/\blang(?:uage)?-([\w-]+)\b/i,M=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,C="
",I={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0};return e.highlight=f,e.highlightAuto=g,e.fixMarkup=p,e.highlightBlock=d,e.configure=b,e.initHighlighting=v,e.initHighlightingOnLoad=m,e.registerLanguage=N,e.listLanguages=R,e.getLanguage=w,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},e.C=function(n,t,r){var a=e.inherit({cN:"comment",b:n,e:t,c:[]},r||{});return a.c.push(e.PWM),a.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),a},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e});hljs.registerLanguage("xml",function(s){var e="[A-Za-z0-9\\._:-]+",t={eW:!0,i:/`]+/}]}]}]};return{aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist"],cI:!0,c:[{cN:"meta",b:"",r:10,c:[{b:"\\[",e:"\\]"}]},s.C("",{r:10}),{b:"<\\!\\[CDATA\\[",e:"\\]\\]>",r:10},{b:/<\?(php)?/,e:/\?>/,sL:"php",c:[{b:"/\\*",e:"\\*/",skip:!0}]},{cN:"tag",b:"|$)",e:">",k:{name:"style"},c:[t],starts:{e:"",rE:!0,sL:["css","xml"]}},{cN:"tag",b:"|$)",e:">",k:{name:"script"},c:[t],starts:{e:"",rE:!0,sL:["actionscript","javascript","handlebars","xml"]}},{cN:"meta",v:[{b:/<\?xml/,e:/\?>/,r:10},{b:/<\?\w+/,e:/\?>/}]},{cN:"tag",b:"",c:[{cN:"name",b:/[^\/><\s]+/,r:0},t]}]}});hljs.registerLanguage("markdown",function(e){return{aliases:["md","mkdown","mkd"],c:[{cN:"section",v:[{b:"^#{1,6}",e:"$"},{b:"^.+?\\n[=-]{2,}$"}]},{b:"<",e:">",sL:"xml",r:0},{cN:"bullet",b:"^([*+-]|(\\d+\\.))\\s+"},{cN:"strong",b:"[*_]{2}.+?[*_]{2}"},{cN:"emphasis",v:[{b:"\\*.+?\\*"},{b:"_.+?_",r:0}]},{cN:"quote",b:"^>\\s+",e:"$"},{cN:"code",v:[{b:"^```w*s*$",e:"^```s*$"},{b:"`.+?`"},{b:"^( {4}| )",e:"$",r:0}]},{b:"^[-\\*]{3,}",e:"$"},{b:"\\[.+?\\][\\(\\[].*?[\\)\\]]",rB:!0,c:[{cN:"string",b:"\\[",e:"\\]",eB:!0,rE:!0,r:0},{cN:"link",b:"\\]\\(",e:"\\)",eB:!0,eE:!0},{cN:"symbol",b:"\\]\\[",e:"\\]",eB:!0,eE:!0}],r:10},{b:/^\[[^\n]+\]:/,rB:!0,c:[{cN:"symbol",b:/\[/,e:/\]/,eB:!0,eE:!0},{cN:"link",b:/:\s*/,e:/$/,eB:!0}]}]}});hljs.registerLanguage("bash",function(e){var t={cN:"variable",v:[{b:/\$[\w\d#@][\w\d_]*/},{b:/\$\{(.*?)}/}]},s={cN:"string",b:/"/,e:/"/,c:[e.BE,t,{cN:"variable",b:/\$\(/,e:/\)/,c:[e.BE]}]},a={cN:"string",b:/'/,e:/'/};return{aliases:["sh","zsh"],l:/-?[a-z\._]+/,k:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},c:[{cN:"meta",b:/^#![^\n]+sh\s*$/,r:10},{cN:"function",b:/\w[\w\d_]*\s*\(\s*\)\s*\{/,rB:!0,c:[e.inherit(e.TM,{b:/\w[\w\d_]*/})],r:0},e.HCM,s,a,t]}});hljs.registerLanguage("dockerfile",function(e){return{aliases:["docker"],cI:!0,k:"from maintainer expose env arg user onbuild stopsignal",c:[e.HCM,e.ASM,e.QSM,e.NM,{bK:"run cmd entrypoint volume add copy workdir label healthcheck shell",starts:{e:/[^\\]\n/,sL:"bash"}}],i:"\\&\\#][a-zA-Z0-9_\\-\\+\\*\\/\\<\\=\\>\\&\\#!]*",c="\\|[^]*?\\|",r="(\\-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s|D|E|F|L|S)(\\+|\\-)?\\d+)?",a={cN:"meta",b:"^#!",e:"$"},l={cN:"literal",b:"\\b(t{1}|nil)\\b"},n={cN:"number",v:[{b:r,r:0},{b:"#(b|B)[0-1]+(/[0-1]+)?"},{b:"#(o|O)[0-7]+(/[0-7]+)?"},{b:"#(x|X)[0-9a-fA-F]+(/[0-9a-fA-F]+)?"},{b:"#(c|C)\\("+r+" +"+r,e:"\\)"}]},i=b.inherit(b.QSM,{i:null}),t=b.C(";","$",{r:0}),s={b:"\\*",e:"\\*"},u={cN:"symbol",b:"[:&]"+e},d={b:e,r:0},f={b:c},m={b:"\\(",e:"\\)",c:["self",l,i,n,d]},o={c:[n,i,s,u,m,d],v:[{b:"['`]\\(",e:"\\)"},{b:"\\(quote ",e:"\\)",k:{name:"quote"}},{b:"'"+c}]},v={v:[{b:"'"+e},{b:"#'"+e+"(::"+e+")*"}]},N={b:"\\(\\s*",e:"\\)"},A={eW:!0,r:0};return N.c=[{cN:"name",v:[{b:e},{b:c}]},A],A.c=[o,v,N,l,n,i,t,s,u,f,d],{i:/\S/,c:[n,a,l,i,t,o,v,N,d]}});hljs.registerLanguage("css",function(e){var c="[a-zA-Z-][a-zA-Z0-9_-]*",t={b:/[A-Z\_\.\-]+\s*:/,rB:!0,e:";",eW:!0,c:[{cN:"attribute",b:/\S/,e:":",eE:!0,starts:{eW:!0,eE:!0,c:[{b:/[\w-]+\(/,rB:!0,c:[{cN:"built_in",b:/[\w-]+/},{b:/\(/,e:/\)/,c:[e.ASM,e.QSM]}]},e.CSSNM,e.QSM,e.ASM,e.CBCM,{cN:"number",b:"#[0-9A-Fa-f]+"},{cN:"meta",b:"!important"}]}}]};return{cI:!0,i:/[=\/|'\$]/,c:[e.CBCM,{cN:"selector-id",b:/#[A-Za-z0-9_-]+/},{cN:"selector-class",b:/\.[A-Za-z0-9_-]+/},{cN:"selector-attr",b:/\[/,e:/\]/,i:"$"},{cN:"selector-pseudo",b:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{b:"@(font-face|page)",l:"[a-z-]+",k:"font-face page"},{b:"@",e:"[{;]",i:/:/,c:[{cN:"keyword",b:/\w+/},{b:/\s/,eW:!0,eE:!0,r:0,c:[e.ASM,e.QSM,e.CSSNM]}]},{cN:"selector-tag",b:c,r:0},{b:"{",e:"}",i:/\S/,c:[e.CBCM,t]}]}});hljs.registerLanguage("makefile",function(e){var i={cN:"variable",v:[{b:"\\$\\("+e.UIR+"\\)",c:[e.BE]},{b:/\$[@%%$#]",starts:{e:"$",sL:"bash"}}]}});hljs.registerLanguage("elixir",function(e){var r="[a-zA-Z_][a-zA-Z0-9_]*(\\!|\\?)?",n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",b="and false then defined module in return redo retry end for true self when next until do begin unless nil break not case cond alias while ensure or include use alias fn quote",c={cN:"subst",b:"#\\{",e:"}",l:r,k:b},a={cN:"string",c:[e.BE,c],v:[{b:/'/,e:/'/},{b:/"/,e:/"/}]},i={cN:"function",bK:"def defp defmacro",e:/\B\b/,c:[e.inherit(e.TM,{b:r,endsParent:!0})]},l=e.inherit(i,{cN:"class",bK:"defimpl defmodule defprotocol defrecord",e:/\bdo\b|$|;/}),s=[a,e.HCM,l,i,{cN:"symbol",b:":(?!\\s)",c:[a,{b:n}],r:0},{cN:"symbol",b:r+":",r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{cN:"variable",b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{b:"->"},{b:"("+e.RSR+")\\s*",c:[e.HCM,{cN:"regexp",i:"\\n",c:[e.BE,c],v:[{b:"/",e:"/[a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}],r:0}];return c.c=s,{l:r,k:b,c:s}});hljs.registerLanguage("erlang",function(e){var r="[a-z'][a-zA-Z0-9_']*",c="("+r+":"+r+"|"+r+")",b={keyword:"after and andalso|10 band begin bnot bor bsl bzr bxor case catch cond div end fun if let not of orelse|10 query receive rem try when xor",literal:"false true"},i=e.C("%","$"),n={cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},a={b:"fun\\s+"+r+"/\\d+"},d={b:c+"\\(",e:"\\)",rB:!0,r:0,c:[{b:c,r:0},{b:"\\(",e:"\\)",eW:!0,rE:!0,r:0}]},o={b:"{",e:"}",r:0},t={b:"\\b_([A-Z][A-Za-z0-9_]*)?",r:0},f={b:"[A-Z][a-zA-Z0-9_]*",r:0},l={b:"#"+e.UIR,r:0,rB:!0,c:[{b:"#"+e.UIR,r:0},{b:"{",e:"}",r:0}]},s={bK:"fun receive if try case",e:"end",k:b};s.c=[i,a,e.inherit(e.ASM,{cN:""}),s,d,e.QSM,n,o,t,f,l];var u=[i,a,s,d,e.QSM,n,o,t,f,l];d.c[1].c=u,o.c=u,l.c[1].c=u;var h={cN:"params",b:"\\(",e:"\\)",c:u};return{aliases:["erl"],k:b,i:"(",rB:!0,i:"\\(|#|//|/\\*|\\\\|:|;",c:[h,e.inherit(e.TM,{b:r})],starts:{e:";|\\.",k:b,c:u}},i,{b:"^-",e:"\\.",r:0,eE:!0,rB:!0,l:"-"+e.IR,k:"-module -record -undef -export -ifdef -ifndef -author -copyright -doc -vsn -import -include -include_lib -compile -define -else -endif -file -behaviour -behavior -spec",c:[h]},n,e.QSM,l,t,f,o,{b:/\.$/}]}});hljs.registerLanguage("scheme",function(e){var t="[^\\(\\)\\[\\]\\{\\}\",'`;#|\\\\\\s]+",r="(\\-|\\+)?\\d+([./]\\d+)?",a=r+"[+\\-]"+r+"i",i={"builtin-name":"case-lambda call/cc class define-class exit-handler field import inherit init-field interface let*-values let-values let/ec mixin opt-lambda override protect provide public rename require require-for-syntax syntax syntax-case syntax-error unit/sig unless when with-syntax and begin call-with-current-continuation call-with-input-file call-with-output-file case cond define define-syntax delay do dynamic-wind else for-each if lambda let let* let-syntax letrec letrec-syntax map or syntax-rules ' * + , ,@ - ... / ; < <= = => > >= ` abs acos angle append apply asin assoc assq assv atan boolean? caar cadr call-with-input-file call-with-output-file call-with-values car cdddar cddddr cdr ceiling char->integer char-alphabetic? char-ci<=? char-ci=? char-ci>? char-downcase char-lower-case? char-numeric? char-ready? char-upcase char-upper-case? char-whitespace? char<=? char=? char>? char? close-input-port close-output-port complex? cons cos current-input-port current-output-port denominator display eof-object? eq? equal? eqv? eval even? exact->inexact exact? exp expt floor force gcd imag-part inexact->exact inexact? input-port? integer->char integer? interaction-environment lcm length list list->string list->vector list-ref list-tail list? load log magnitude make-polar make-rectangular make-string make-vector max member memq memv min modulo negative? newline not null-environment null? number->string number? numerator odd? open-input-file open-output-file output-port? pair? peek-char port? positive? procedure? quasiquote quote quotient rational? rationalize read read-char real-part real? remainder reverse round scheme-report-environment set! set-car! set-cdr! sin sqrt string string->list string->number string->symbol string-append string-ci<=? string-ci=? string-ci>? string-copy string-fill! string-length string-ref string-set! string<=? string=? string>? string? substring symbol->string symbol? tan transcript-off transcript-on truncate values vector vector->list vector-fill! vector-length vector-ref vector-set! with-input-from-file with-output-to-file write write-char zero?"},n={cN:"meta",b:"^#!",e:"$"},c={cN:"literal",b:"(#t|#f|#\\\\"+t+"|#\\\\.)"},l={cN:"number",v:[{b:r,r:0},{b:a,r:0},{b:"#b[0-1]+(/[0-1]+)?"},{b:"#o[0-7]+(/[0-7]+)?"},{b:"#x[0-9a-f]+(/[0-9a-f]+)?"}]},s=e.QSM,o=[e.C(";","$",{r:0}),e.C("#\\|","\\|#")],u={b:t,r:0},p={cN:"symbol",b:"'"+t},d={eW:!0,r:0},m={v:[{b:/'/},{b:"`"}],c:[{b:"\\(",e:"\\)",c:["self",c,s,l,u,p]}]},g={cN:"name",b:t,l:t,k:i},h={b:/lambda/,eW:!0,rB:!0,c:[g,{b:/\(/,e:/\)/,endsParent:!0,c:[u]}]},b={v:[{b:"\\(",e:"\\)"},{b:"\\[",e:"\\]"}],c:[h,g,d]};return d.c=[c,l,s,u,p,m,b].concat(o),{i:/\S/,c:[n,l,s,p,m,b].concat(o)}});hljs.registerLanguage("clojure",function(e){var t={"builtin-name":"def defonce cond apply if-not if-let if not not= = < > <= >= == + / * - rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit defmacro defn defn- macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},r="a-zA-Z_\\-!.?+*=<>&#'",n="["+r+"]["+r+"0-9/;:]*",a="[-+]?\\d+(\\.\\d+)?",o={b:n,r:0},s={cN:"number",b:a,r:0},i=e.inherit(e.QSM,{i:null}),c=e.C(";","$",{r:0}),d={cN:"literal",b:/\b(true|false|nil)\b/},l={b:"[\\[\\{]",e:"[\\]\\}]"},m={cN:"comment",b:"\\^"+n},p=e.C("\\^\\{","\\}"),u={cN:"symbol",b:"[:]{1,2}"+n},f={b:"\\(",e:"\\)"},h={eW:!0,r:0},y={k:t,l:n,cN:"name",b:n,starts:h},b=[f,i,m,p,c,u,l,s,d,o];return f.c=[e.C("comment",""),y,h],h.c=b,l.c=b,{aliases:["clj"],i:/\S/,c:[f,i,m,p,c,u,l,s,d]}});hljs.registerLanguage("clojure-repl",function(e){return{c:[{cN:"meta",b:/^([\w.-]+|\s*#_)=>/,starts:{e:/$/,sL:"clojure"}}]}});hljs.registerLanguage("ruby",function(e){var b="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",r={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},c={cN:"doctag",b:"@[A-Za-z]+"},a={b:"#<",e:">"},s=[e.C("#","$",{c:[c]}),e.C("^\\=begin","^\\=end",{c:[c],r:10}),e.C("^__END__","\\n$")],n={cN:"subst",b:"#\\{",e:"}",k:r},t={cN:"string",c:[e.BE,n],v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/`/,e:/`/},{b:"%[qQwWx]?\\(",e:"\\)"},{b:"%[qQwWx]?\\[",e:"\\]"},{b:"%[qQwWx]?{",e:"}"},{b:"%[qQwWx]?<",e:">"},{b:"%[qQwWx]?/",e:"/"},{b:"%[qQwWx]?%",e:"%"},{b:"%[qQwWx]?-",e:"-"},{b:"%[qQwWx]?\\|",e:"\\|"},{b:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{b:/<<(-?)\w+$/,e:/^\s*\w+$/}]},i={cN:"params",b:"\\(",e:"\\)",endsParent:!0,k:r},d=[t,a,{cN:"class",bK:"class module",e:"$|;",i:/=/,c:[e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{b:"<\\s*",c:[{b:"("+e.IR+"::)?"+e.IR}]}].concat(s)},{cN:"function",bK:"def",e:"$|;",c:[e.inherit(e.TM,{b:b}),i].concat(s)},{b:e.IR+"::"},{cN:"symbol",b:e.UIR+"(\\!|\\?)?:",r:0},{cN:"symbol",b:":(?!\\s)",c:[t,{b:b}],r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{cN:"params",b:/\|/,e:/\|/,k:r},{b:"("+e.RSR+"|unless)\\s*",k:"unless",c:[a,{cN:"regexp",c:[e.BE,n],i:/\n/,v:[{b:"/",e:"/[a-z]*"},{b:"%r{",e:"}[a-z]*"},{b:"%r\\(",e:"\\)[a-z]*"},{b:"%r!",e:"![a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}].concat(s),r:0}].concat(s);n.c=d,i.c=d;var l="[>?]>",o="[\\w#]+\\(\\w+\\):\\d+:\\d+>",u="(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>",w=[{b:/^\s*=>/,starts:{e:"$",c:d}},{cN:"meta",b:"^("+l+"|"+o+"|"+u+")",starts:{e:"$",c:d}}];return{aliases:["rb","gemspec","podspec","thor","irb"],k:r,i:/\/\*/,c:s.concat(w).concat(d)}});hljs.registerLanguage("yaml",function(e){var b="true false yes no null",a="^[ \\-]*",r="[a-zA-Z_][\\w\\-]*",t={cN:"attr",v:[{b:a+r+":"},{b:a+'"'+r+'":'},{b:a+"'"+r+"':"}]},c={cN:"template-variable",v:[{b:"{{",e:"}}"},{b:"%{",e:"}"}]},l={cN:"string",r:0,v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/\S+/}],c:[e.BE,c]};return{cI:!0,aliases:["yml","YAML","yaml"],c:[t,{cN:"meta",b:"^---s*$",r:10},{cN:"string",b:"[\\|>] *$",rE:!0,c:l.c,e:t.v[0].b},{b:"<%[%=-]?",e:"[%-]?%>",sL:"ruby",eB:!0,eE:!0,r:0},{cN:"type",b:"!!"+e.UIR},{cN:"meta",b:"&"+e.UIR+"$"},{cN:"meta",b:"\\*"+e.UIR+"$"},{cN:"bullet",b:"^ *-",r:0},e.HCM,{bK:b,k:{literal:b}},e.CNM,l]}});hljs.registerLanguage("erlang-repl",function(e){return{k:{built_in:"spawn spawn_link self",keyword:"after and andalso|10 band begin bnot bor bsl bsr bxor case catch cond div end fun if let not of or orelse|10 query receive rem try when xor"},c:[{cN:"meta",b:"^[0-9]+> ",r:10},e.C("%","$"),{cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},e.ASM,e.QSM,{b:"\\?(::)?([A-Z]\\w*(::)?)+"},{b:"->"},{b:"ok"},{b:"!"},{b:"(\\b[a-z'][a-zA-Z0-9_']*:[a-z'][a-zA-Z0-9_']*)|(\\b[a-z'][a-zA-Z0-9_']*)",r:0},{b:"[A-Z][a-zA-Z0-9_']*",r:0}]}}); \ No newline at end of file diff --git a/docs/current/highlight/solarized-light.css b/docs/current/highlight/solarized-light.css new file mode 100644 index 0000000..fdcfcc7 --- /dev/null +++ b/docs/current/highlight/solarized-light.css @@ -0,0 +1,84 @@ +/* + +Orginal Style from ethanschoonover.com/solarized (c) Jeremy Hull + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + background: #fdf6e3; + color: #657b83; +} + +.hljs-comment, +.hljs-quote { + color: #93a1a1; +} + +/* Solarized Green */ +.hljs-keyword, +.hljs-selector-tag, +.hljs-addition { + color: #859900; +} + +/* Solarized Cyan */ +.hljs-number, +.hljs-string, +.hljs-meta .hljs-meta-string, +.hljs-literal, +.hljs-doctag, +.hljs-regexp { + color: #2aa198; +} + +/* Solarized Blue */ +.hljs-title, +.hljs-section, +.hljs-name, +.hljs-selector-id, +.hljs-selector-class { + color: #268bd2; +} + +/* Solarized Yellow */ +.hljs-attribute, +.hljs-attr, +.hljs-variable, +.hljs-template-variable, +.hljs-class .hljs-title, +.hljs-type { + color: #b58900; +} + +/* Solarized Orange */ +.hljs-symbol, +.hljs-bullet, +.hljs-subst, +.hljs-meta, +.hljs-meta .hljs-keyword, +.hljs-selector-attr, +.hljs-selector-pseudo, +.hljs-link { + color: #cb4b16; +} + +/* Solarized Red */ +.hljs-built_in, +.hljs-deletion { + color: #dc322f; +} + +.hljs-formula { + background: #eee8d5; +} + +.hljs-emphasis { + font-style: italic; +} + +.hljs-strong { + font-weight: bold; +} diff --git a/docs/current/index.html b/docs/current/index.html new file mode 100644 index 0000000..298162c --- /dev/null +++ b/docs/current/index.html @@ -0,0 +1,3 @@ + +protobuf 3.4.0-v0.3-SNAPSHOT

protobuf 3.4.0-v0.3-SNAPSHOT

Released under the Eclipse Public License

A Clojure interface to Google's protocol buffers.

Installation

To install, add the following dependency to your project or build file:

[clojusc/protobuf "3.4.0-v0.3-SNAPSHOT"]

Namespaces

flatland.protobuf.codec

Documentation forthcoming

Public variables and functions:

flatland.protobuf.schema

Documentation forthcoming

\ No newline at end of file diff --git a/docs/current/js/jquery.min.js b/docs/current/js/jquery.min.js new file mode 100644 index 0000000..73f33fb --- /dev/null +++ b/docs/current/js/jquery.min.js @@ -0,0 +1,4 @@ +/*! jQuery v1.11.0 | (c) 2005, 2014 jQuery Foundation, Inc. | jquery.org/license */ +!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=c.slice,e=c.concat,f=c.push,g=c.indexOf,h={},i=h.toString,j=h.hasOwnProperty,k="".trim,l={},m="1.11.0",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return d.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:d.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a,b){return n.each(this,a,b)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(d.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor(null)},push:f,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(n.isPlainObject(c)||(b=n.isArray(c)))?(b?(b=!1,f=a&&n.isArray(a)?a:[]):f=a&&n.isPlainObject(a)?a:{},g[d]=n.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray||function(a){return"array"===n.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){return a-parseFloat(a)>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==n.type(a)||a.nodeType||n.isWindow(a))return!1;try{if(a.constructor&&!j.call(a,"constructor")&&!j.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(l.ownLast)for(b in a)return j.call(a,b);for(b in a);return void 0===b||j.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?h[i.call(a)]||"object":typeof a},globalEval:function(b){b&&n.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b,c){var d,e=0,f=a.length,g=s(a);if(c){if(g){for(;f>e;e++)if(d=b.apply(a[e],c),d===!1)break}else for(e in a)if(d=b.apply(a[e],c),d===!1)break}else if(g){for(;f>e;e++)if(d=b.call(a[e],e,a[e]),d===!1)break}else for(e in a)if(d=b.call(a[e],e,a[e]),d===!1)break;return a},trim:k&&!k.call("\ufeff\xa0")?function(a){return null==a?"":k.call(a)}:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):f.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(g)return g.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,f=0,g=a.length,h=s(a),i=[];if(h)for(;g>f;f++)d=b(a[f],f,c),null!=d&&i.push(d);else for(f in a)d=b(a[f],f,c),null!=d&&i.push(d);return e.apply([],i)},guid:1,proxy:function(a,b){var c,e,f;return"string"==typeof b&&(f=a[b],b=a,a=f),n.isFunction(a)?(c=d.call(arguments,2),e=function(){return a.apply(b||this,c.concat(d.call(arguments)))},e.guid=a.guid=a.guid||n.guid++,e):void 0},now:function(){return+new Date},support:l}),n.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(a,b){h["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:1===a.nodeType&&b?!0:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s="sizzle"+-new Date,t=a.document,u=0,v=0,w=eb(),x=eb(),y=eb(),z=function(a,b){return a===b&&(j=!0),0},A="undefined",B=1<<31,C={}.hasOwnProperty,D=[],E=D.pop,F=D.push,G=D.push,H=D.slice,I=D.indexOf||function(a){for(var b=0,c=this.length;c>b;b++)if(this[b]===a)return b;return-1},J="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",K="[\\x20\\t\\r\\n\\f]",L="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",M=L.replace("w","w#"),N="\\["+K+"*("+L+")"+K+"*(?:([*^$|!~]?=)"+K+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+M+")|)|)"+K+"*\\]",O=":("+L+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+N.replace(3,8)+")*)|.*)\\)|)",P=new RegExp("^"+K+"+|((?:^|[^\\\\])(?:\\\\.)*)"+K+"+$","g"),Q=new RegExp("^"+K+"*,"+K+"*"),R=new RegExp("^"+K+"*([>+~]|"+K+")"+K+"*"),S=new RegExp("="+K+"*([^\\]'\"]*?)"+K+"*\\]","g"),T=new RegExp(O),U=new RegExp("^"+M+"$"),V={ID:new RegExp("^#("+L+")"),CLASS:new RegExp("^\\.("+L+")"),TAG:new RegExp("^("+L.replace("w","w*")+")"),ATTR:new RegExp("^"+N),PSEUDO:new RegExp("^"+O),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+K+"*(even|odd|(([+-]|)(\\d*)n|)"+K+"*(?:([+-]|)"+K+"*(\\d+)|))"+K+"*\\)|)","i"),bool:new RegExp("^(?:"+J+")$","i"),needsContext:new RegExp("^"+K+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+K+"*((?:-\\d)?\\d*)"+K+"*\\)|)(?=[^-]|$)","i")},W=/^(?:input|select|textarea|button)$/i,X=/^h\d$/i,Y=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,$=/[+~]/,_=/'|\\/g,ab=new RegExp("\\\\([\\da-f]{1,6}"+K+"?|("+K+")|.)","ig"),bb=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)};try{G.apply(D=H.call(t.childNodes),t.childNodes),D[t.childNodes.length].nodeType}catch(cb){G={apply:D.length?function(a,b){F.apply(a,H.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function db(a,b,d,e){var f,g,h,i,j,m,p,q,u,v;if((b?b.ownerDocument||b:t)!==l&&k(b),b=b||l,d=d||[],!a||"string"!=typeof a)return d;if(1!==(i=b.nodeType)&&9!==i)return[];if(n&&!e){if(f=Z.exec(a))if(h=f[1]){if(9===i){if(g=b.getElementById(h),!g||!g.parentNode)return d;if(g.id===h)return d.push(g),d}else if(b.ownerDocument&&(g=b.ownerDocument.getElementById(h))&&r(b,g)&&g.id===h)return d.push(g),d}else{if(f[2])return G.apply(d,b.getElementsByTagName(a)),d;if((h=f[3])&&c.getElementsByClassName&&b.getElementsByClassName)return G.apply(d,b.getElementsByClassName(h)),d}if(c.qsa&&(!o||!o.test(a))){if(q=p=s,u=b,v=9===i&&a,1===i&&"object"!==b.nodeName.toLowerCase()){m=ob(a),(p=b.getAttribute("id"))?q=p.replace(_,"\\$&"):b.setAttribute("id",q),q="[id='"+q+"'] ",j=m.length;while(j--)m[j]=q+pb(m[j]);u=$.test(a)&&mb(b.parentNode)||b,v=m.join(",")}if(v)try{return G.apply(d,u.querySelectorAll(v)),d}catch(w){}finally{p||b.removeAttribute("id")}}}return xb(a.replace(P,"$1"),b,d,e)}function eb(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function fb(a){return a[s]=!0,a}function gb(a){var b=l.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function hb(a,b){var c=a.split("|"),e=a.length;while(e--)d.attrHandle[c[e]]=b}function ib(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||B)-(~a.sourceIndex||B);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function jb(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function kb(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function lb(a){return fb(function(b){return b=+b,fb(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function mb(a){return a&&typeof a.getElementsByTagName!==A&&a}c=db.support={},f=db.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},k=db.setDocument=function(a){var b,e=a?a.ownerDocument||a:t,g=e.defaultView;return e!==l&&9===e.nodeType&&e.documentElement?(l=e,m=e.documentElement,n=!f(e),g&&g!==g.top&&(g.addEventListener?g.addEventListener("unload",function(){k()},!1):g.attachEvent&&g.attachEvent("onunload",function(){k()})),c.attributes=gb(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=gb(function(a){return a.appendChild(e.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Y.test(e.getElementsByClassName)&&gb(function(a){return a.innerHTML="
",a.firstChild.className="i",2===a.getElementsByClassName("i").length}),c.getById=gb(function(a){return m.appendChild(a).id=s,!e.getElementsByName||!e.getElementsByName(s).length}),c.getById?(d.find.ID=function(a,b){if(typeof b.getElementById!==A&&n){var c=b.getElementById(a);return c&&c.parentNode?[c]:[]}},d.filter.ID=function(a){var b=a.replace(ab,bb);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(ab,bb);return function(a){var c=typeof a.getAttributeNode!==A&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return typeof b.getElementsByTagName!==A?b.getElementsByTagName(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return typeof b.getElementsByClassName!==A&&n?b.getElementsByClassName(a):void 0},p=[],o=[],(c.qsa=Y.test(e.querySelectorAll))&&(gb(function(a){a.innerHTML="",a.querySelectorAll("[t^='']").length&&o.push("[*^$]="+K+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||o.push("\\["+K+"*(?:value|"+J+")"),a.querySelectorAll(":checked").length||o.push(":checked")}),gb(function(a){var b=e.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&o.push("name"+K+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||o.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),o.push(",.*:")})),(c.matchesSelector=Y.test(q=m.webkitMatchesSelector||m.mozMatchesSelector||m.oMatchesSelector||m.msMatchesSelector))&&gb(function(a){c.disconnectedMatch=q.call(a,"div"),q.call(a,"[s!='']:x"),p.push("!=",O)}),o=o.length&&new RegExp(o.join("|")),p=p.length&&new RegExp(p.join("|")),b=Y.test(m.compareDocumentPosition),r=b||Y.test(m.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},z=b?function(a,b){if(a===b)return j=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===e||a.ownerDocument===t&&r(t,a)?-1:b===e||b.ownerDocument===t&&r(t,b)?1:i?I.call(i,a)-I.call(i,b):0:4&d?-1:1)}:function(a,b){if(a===b)return j=!0,0;var c,d=0,f=a.parentNode,g=b.parentNode,h=[a],k=[b];if(!f||!g)return a===e?-1:b===e?1:f?-1:g?1:i?I.call(i,a)-I.call(i,b):0;if(f===g)return ib(a,b);c=a;while(c=c.parentNode)h.unshift(c);c=b;while(c=c.parentNode)k.unshift(c);while(h[d]===k[d])d++;return d?ib(h[d],k[d]):h[d]===t?-1:k[d]===t?1:0},e):l},db.matches=function(a,b){return db(a,null,null,b)},db.matchesSelector=function(a,b){if((a.ownerDocument||a)!==l&&k(a),b=b.replace(S,"='$1']"),!(!c.matchesSelector||!n||p&&p.test(b)||o&&o.test(b)))try{var d=q.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return db(b,l,null,[a]).length>0},db.contains=function(a,b){return(a.ownerDocument||a)!==l&&k(a),r(a,b)},db.attr=function(a,b){(a.ownerDocument||a)!==l&&k(a);var e=d.attrHandle[b.toLowerCase()],f=e&&C.call(d.attrHandle,b.toLowerCase())?e(a,b,!n):void 0;return void 0!==f?f:c.attributes||!n?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},db.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},db.uniqueSort=function(a){var b,d=[],e=0,f=0;if(j=!c.detectDuplicates,i=!c.sortStable&&a.slice(0),a.sort(z),j){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return i=null,a},e=db.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=db.selectors={cacheLength:50,createPseudo:fb,match:V,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(ab,bb),a[3]=(a[4]||a[5]||"").replace(ab,bb),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||db.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&db.error(a[0]),a},PSEUDO:function(a){var b,c=!a[5]&&a[2];return V.CHILD.test(a[0])?null:(a[3]&&void 0!==a[4]?a[2]=a[4]:c&&T.test(c)&&(b=ob(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(ab,bb).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=w[a+" "];return b||(b=new RegExp("(^|"+K+")"+a+"("+K+"|$)"))&&w(a,function(a){return b.test("string"==typeof a.className&&a.className||typeof a.getAttribute!==A&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=db.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),t=!i&&!h;if(q){if(f){while(p){l=b;while(l=l[p])if(h?l.nodeName.toLowerCase()===r:1===l.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&t){k=q[s]||(q[s]={}),j=k[a]||[],n=j[0]===u&&j[1],m=j[0]===u&&j[2],l=n&&q.childNodes[n];while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if(1===l.nodeType&&++m&&l===b){k[a]=[u,n,m];break}}else if(t&&(j=(b[s]||(b[s]={}))[a])&&j[0]===u)m=j[1];else while(l=++n&&l&&l[p]||(m=n=0)||o.pop())if((h?l.nodeName.toLowerCase()===r:1===l.nodeType)&&++m&&(t&&((l[s]||(l[s]={}))[a]=[u,m]),l===b))break;return m-=e,m===d||m%d===0&&m/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||db.error("unsupported pseudo: "+a);return e[s]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?fb(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=I.call(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:fb(function(a){var b=[],c=[],d=g(a.replace(P,"$1"));return d[s]?fb(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),!c.pop()}}),has:fb(function(a){return function(b){return db(a,b).length>0}}),contains:fb(function(a){return function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:fb(function(a){return U.test(a||"")||db.error("unsupported lang: "+a),a=a.replace(ab,bb).toLowerCase(),function(b){var c;do if(c=n?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===m},focus:function(a){return a===l.activeElement&&(!l.hasFocus||l.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return X.test(a.nodeName)},input:function(a){return W.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:lb(function(){return[0]}),last:lb(function(a,b){return[b-1]}),eq:lb(function(a,b,c){return[0>c?c+b:c]}),even:lb(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:lb(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:lb(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:lb(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function qb(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=v++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j=[u,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(i=b[s]||(b[s]={}),(h=i[d])&&h[0]===u&&h[1]===f)return j[2]=h[2];if(i[d]=j,j[2]=a(b,c,g))return!0}}}function rb(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function sb(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(!c||c(f,d,e))&&(g.push(f),j&&b.push(h));return g}function tb(a,b,c,d,e,f){return d&&!d[s]&&(d=tb(d)),e&&!e[s]&&(e=tb(e,f)),fb(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||wb(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:sb(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=sb(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?I.call(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=sb(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):G.apply(g,r)})}function ub(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],i=g||d.relative[" "],j=g?1:0,k=qb(function(a){return a===b},i,!0),l=qb(function(a){return I.call(b,a)>-1},i,!0),m=[function(a,c,d){return!g&&(d||c!==h)||((b=c).nodeType?k(a,c,d):l(a,c,d))}];f>j;j++)if(c=d.relative[a[j].type])m=[qb(rb(m),c)];else{if(c=d.filter[a[j].type].apply(null,a[j].matches),c[s]){for(e=++j;f>e;e++)if(d.relative[a[e].type])break;return tb(j>1&&rb(m),j>1&&pb(a.slice(0,j-1).concat({value:" "===a[j-2].type?"*":""})).replace(P,"$1"),c,e>j&&ub(a.slice(j,e)),f>e&&ub(a=a.slice(e)),f>e&&pb(a))}m.push(c)}return rb(m)}function vb(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,i,j,k){var m,n,o,p=0,q="0",r=f&&[],s=[],t=h,v=f||e&&d.find.TAG("*",k),w=u+=null==t?1:Math.random()||.1,x=v.length;for(k&&(h=g!==l&&g);q!==x&&null!=(m=v[q]);q++){if(e&&m){n=0;while(o=a[n++])if(o(m,g,i)){j.push(m);break}k&&(u=w)}c&&((m=!o&&m)&&p--,f&&r.push(m))}if(p+=q,c&&q!==p){n=0;while(o=b[n++])o(r,s,g,i);if(f){if(p>0)while(q--)r[q]||s[q]||(s[q]=E.call(j));s=sb(s)}G.apply(j,s),k&&!f&&s.length>0&&p+b.length>1&&db.uniqueSort(j)}return k&&(u=w,h=t),r};return c?fb(f):f}g=db.compile=function(a,b){var c,d=[],e=[],f=y[a+" "];if(!f){b||(b=ob(a)),c=b.length;while(c--)f=ub(b[c]),f[s]?d.push(f):e.push(f);f=y(a,vb(e,d))}return f};function wb(a,b,c){for(var d=0,e=b.length;e>d;d++)db(a,b[d],c);return c}function xb(a,b,e,f){var h,i,j,k,l,m=ob(a);if(!f&&1===m.length){if(i=m[0]=m[0].slice(0),i.length>2&&"ID"===(j=i[0]).type&&c.getById&&9===b.nodeType&&n&&d.relative[i[1].type]){if(b=(d.find.ID(j.matches[0].replace(ab,bb),b)||[])[0],!b)return e;a=a.slice(i.shift().value.length)}h=V.needsContext.test(a)?0:i.length;while(h--){if(j=i[h],d.relative[k=j.type])break;if((l=d.find[k])&&(f=l(j.matches[0].replace(ab,bb),$.test(i[0].type)&&mb(b.parentNode)||b))){if(i.splice(h,1),a=f.length&&pb(i),!a)return G.apply(e,f),e;break}}}return g(a,m)(f,b,!n,e,$.test(a)&&mb(b.parentNode)||b),e}return c.sortStable=s.split("").sort(z).join("")===s,c.detectDuplicates=!!j,k(),c.sortDetached=gb(function(a){return 1&a.compareDocumentPosition(l.createElement("div"))}),gb(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||hb("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&gb(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||hb("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),gb(function(a){return null==a.getAttribute("disabled")})||hb(J,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),db}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=n.expr.match.needsContext,v=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,w=/^.[^:#\[\.,]*$/;function x(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(w.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return n.inArray(a,b)>=0!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;e>b;b++)if(n.contains(d[b],this))return!0}));for(b=0;e>b;b++)n.find(a,d[b],c);return c=this.pushStack(e>1?n.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(x(this,a||[],!1))},not:function(a){return this.pushStack(x(this,a||[],!0))},is:function(a){return!!x(this,"string"==typeof a&&u.test(a)?n(a):a||[],!1).length}});var y,z=a.document,A=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,B=n.fn.init=function(a,b){var c,d;if(!a)return this;if("string"==typeof a){if(c="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:A.exec(a),!c||!c[1]&&b)return!b||b.jquery?(b||y).find(a):this.constructor(b).find(a);if(c[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(c[1],b&&b.nodeType?b.ownerDocument||b:z,!0)),v.test(c[1])&&n.isPlainObject(b))for(c in b)n.isFunction(this[c])?this[c](b[c]):this.attr(c,b[c]);return this}if(d=z.getElementById(c[2]),d&&d.parentNode){if(d.id!==c[2])return y.find(a);this.length=1,this[0]=d}return this.context=z,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof y.ready?y.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};B.prototype=n.fn,y=n(z);var C=/^(?:parents|prev(?:Until|All))/,D={children:!0,contents:!0,next:!0,prev:!0};n.extend({dir:function(a,b,c){var d=[],e=a[b];while(e&&9!==e.nodeType&&(void 0===c||1!==e.nodeType||!n(e).is(c)))1===e.nodeType&&d.push(e),e=e[b];return d},sibling:function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c}}),n.fn.extend({has:function(a){var b,c=n(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(n.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=u.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.unique(f):f)},index:function(a){return a?"string"==typeof a?n.inArray(this[0],n(a)):n.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.unique(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function E(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return n.dir(a,"parentNode")},parentsUntil:function(a,b,c){return n.dir(a,"parentNode",c)},next:function(a){return E(a,"nextSibling")},prev:function(a){return E(a,"previousSibling")},nextAll:function(a){return n.dir(a,"nextSibling")},prevAll:function(a){return n.dir(a,"previousSibling")},nextUntil:function(a,b,c){return n.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return n.dir(a,"previousSibling",c)},siblings:function(a){return n.sibling((a.parentNode||{}).firstChild,a)},children:function(a){return n.sibling(a.firstChild)},contents:function(a){return n.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(D[a]||(e=n.unique(e)),C.test(a)&&(e=e.reverse())),this.pushStack(e)}});var F=/\S+/g,G={};function H(a){var b=G[a]={};return n.each(a.match(F)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?G[a]||H(a):n.extend({},a);var b,c,d,e,f,g,h=[],i=!a.once&&[],j=function(l){for(c=a.memory&&l,d=!0,f=g||0,g=0,e=h.length,b=!0;h&&e>f;f++)if(h[f].apply(l[0],l[1])===!1&&a.stopOnFalse){c=!1;break}b=!1,h&&(i?i.length&&j(i.shift()):c?h=[]:k.disable())},k={add:function(){if(h){var d=h.length;!function f(b){n.each(b,function(b,c){var d=n.type(c);"function"===d?a.unique&&k.has(c)||h.push(c):c&&c.length&&"string"!==d&&f(c)})}(arguments),b?e=h.length:c&&(g=d,j(c))}return this},remove:function(){return h&&n.each(arguments,function(a,c){var d;while((d=n.inArray(c,h,d))>-1)h.splice(d,1),b&&(e>=d&&e--,f>=d&&f--)}),this},has:function(a){return a?n.inArray(a,h)>-1:!(!h||!h.length)},empty:function(){return h=[],e=0,this},disable:function(){return h=i=c=void 0,this},disabled:function(){return!h},lock:function(){return i=void 0,c||k.disable(),this},locked:function(){return!i},fireWith:function(a,c){return!h||d&&!i||(c=c||[],c=[a,c.slice?c.slice():c],b?i.push(c):j(c)),this},fire:function(){return k.fireWith(this,arguments),this},fired:function(){return!!d}};return k},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().done(c.resolve).fail(c.reject).progress(c.notify):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=d.call(arguments),e=c.length,f=1!==e||a&&n.isFunction(a.promise)?e:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(e){b[a]=this,c[a]=arguments.length>1?d.call(arguments):e,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(e>1)for(i=new Array(e),j=new Array(e),k=new Array(e);e>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().done(h(b,k,c)).fail(g.reject).progress(h(b,j,i)):--f;return f||g.resolveWith(k,c),g.promise()}});var I;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){if(a===!0?!--n.readyWait:!n.isReady){if(!z.body)return setTimeout(n.ready);n.isReady=!0,a!==!0&&--n.readyWait>0||(I.resolveWith(z,[n]),n.fn.trigger&&n(z).trigger("ready").off("ready"))}}});function J(){z.addEventListener?(z.removeEventListener("DOMContentLoaded",K,!1),a.removeEventListener("load",K,!1)):(z.detachEvent("onreadystatechange",K),a.detachEvent("onload",K))}function K(){(z.addEventListener||"load"===event.type||"complete"===z.readyState)&&(J(),n.ready())}n.ready.promise=function(b){if(!I)if(I=n.Deferred(),"complete"===z.readyState)setTimeout(n.ready);else if(z.addEventListener)z.addEventListener("DOMContentLoaded",K,!1),a.addEventListener("load",K,!1);else{z.attachEvent("onreadystatechange",K),a.attachEvent("onload",K);var c=!1;try{c=null==a.frameElement&&z.documentElement}catch(d){}c&&c.doScroll&&!function e(){if(!n.isReady){try{c.doScroll("left")}catch(a){return setTimeout(e,50)}J(),n.ready()}}()}return I.promise(b)};var L="undefined",M;for(M in n(l))break;l.ownLast="0"!==M,l.inlineBlockNeedsLayout=!1,n(function(){var a,b,c=z.getElementsByTagName("body")[0];c&&(a=z.createElement("div"),a.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",b=z.createElement("div"),c.appendChild(a).appendChild(b),typeof b.style.zoom!==L&&(b.style.cssText="border:0;margin:0;width:1px;padding:1px;display:inline;zoom:1",(l.inlineBlockNeedsLayout=3===b.offsetWidth)&&(c.style.zoom=1)),c.removeChild(a),a=b=null)}),function(){var a=z.createElement("div");if(null==l.deleteExpando){l.deleteExpando=!0;try{delete a.test}catch(b){l.deleteExpando=!1}}a=null}(),n.acceptData=function(a){var b=n.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b};var N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(O,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}n.data(a,b,c)}else c=void 0}return c}function Q(a){var b;for(b in a)if(("data"!==b||!n.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function R(a,b,d,e){if(n.acceptData(a)){var f,g,h=n.expando,i=a.nodeType,j=i?n.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||n.guid++:h),j[k]||(j[k]=i?{}:{toJSON:n.noop}),("object"==typeof b||"function"==typeof b)&&(e?j[k]=n.extend(j[k],b):j[k].data=n.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[n.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[n.camelCase(b)])):f=g,f +}}function S(a,b,c){if(n.acceptData(a)){var d,e,f=a.nodeType,g=f?n.cache:a,h=f?a[n.expando]:n.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){n.isArray(b)?b=b.concat(n.map(b,n.camelCase)):b in d?b=[b]:(b=n.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!Q(d):!n.isEmptyObject(d))return}(c||(delete g[h].data,Q(g[h])))&&(f?n.cleanData([a],!0):l.deleteExpando||g!=g.window?delete g[h]:g[h]=null)}}}n.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?n.cache[a[n.expando]]:a[n.expando],!!a&&!Q(a)},data:function(a,b,c){return R(a,b,c)},removeData:function(a,b){return S(a,b)},_data:function(a,b,c){return R(a,b,c,!0)},_removeData:function(a,b){return S(a,b,!0)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=n.data(f),1===f.nodeType&&!n._data(f,"parsedAttrs"))){c=g.length;while(c--)d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d]));n._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){n.data(this,a)}):arguments.length>1?this.each(function(){n.data(this,a,b)}):f?P(f,a,n.data(f,a)):void 0},removeData:function(a){return this.each(function(){n.removeData(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=n._data(a,b),c&&(!d||n.isArray(c)?d=n._data(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return n._data(a,c)||n._data(a,c,{empty:n.Callbacks("once memory").add(function(){n._removeData(a,b+"queue"),n._removeData(a,c)})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},X=/^(?:checkbox|radio)$/i;!function(){var a=z.createDocumentFragment(),b=z.createElement("div"),c=z.createElement("input");if(b.setAttribute("className","t"),b.innerHTML="
a",l.leadingWhitespace=3===b.firstChild.nodeType,l.tbody=!b.getElementsByTagName("tbody").length,l.htmlSerialize=!!b.getElementsByTagName("link").length,l.html5Clone="<:nav>"!==z.createElement("nav").cloneNode(!0).outerHTML,c.type="checkbox",c.checked=!0,a.appendChild(c),l.appendChecked=c.checked,b.innerHTML="",l.noCloneChecked=!!b.cloneNode(!0).lastChild.defaultValue,a.appendChild(b),b.innerHTML="",l.checkClone=b.cloneNode(!0).cloneNode(!0).lastChild.checked,l.noCloneEvent=!0,b.attachEvent&&(b.attachEvent("onclick",function(){l.noCloneEvent=!1}),b.cloneNode(!0).click()),null==l.deleteExpando){l.deleteExpando=!0;try{delete b.test}catch(d){l.deleteExpando=!1}}a=b=c=null}(),function(){var b,c,d=z.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(l[b+"Bubbles"]=c in a)||(d.setAttribute(c,"t"),l[b+"Bubbles"]=d.attributes[c].expando===!1);d=null}();var Y=/^(?:input|select|textarea)$/i,Z=/^key/,$=/^(?:mouse|contextmenu)|click/,_=/^(?:focusinfocus|focusoutblur)$/,ab=/^([^.]*)(?:\.(.+)|)$/;function bb(){return!0}function cb(){return!1}function db(){try{return z.activeElement}catch(a){}}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=n.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return typeof n===L||a&&n.event.triggered===a.type?void 0:n.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(F)||[""],h=b.length;while(h--)f=ab.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=n.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=n.event.special[o]||{},l=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},i),(m=g[o])||(m=g[o]=[],m.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,l):m.push(l),n.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n.hasData(a)&&n._data(a);if(r&&(k=r.events)){b=(b||"").match(F)||[""],j=b.length;while(j--)if(h=ab.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=m.length;while(f--)g=m[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(m.splice(f,1),g.selector&&m.delegateCount--,l.remove&&l.remove.call(a,g));i&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(k)&&(delete r.handle,n._removeData(a,"events"))}},trigger:function(b,c,d,e){var f,g,h,i,k,l,m,o=[d||z],p=j.call(b,"type")?b.type:b,q=j.call(b,"namespace")?b.namespace.split("."):[];if(h=l=d=d||z,3!==d.nodeType&&8!==d.nodeType&&!_.test(p+n.event.triggered)&&(p.indexOf(".")>=0&&(q=p.split("."),p=q.shift(),q.sort()),g=p.indexOf(":")<0&&"on"+p,b=b[n.expando]?b:new n.Event(p,"object"==typeof b&&b),b.isTrigger=e?2:3,b.namespace=q.join("."),b.namespace_re=b.namespace?new RegExp("(^|\\.)"+q.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=d),c=null==c?[b]:n.makeArray(c,[b]),k=n.event.special[p]||{},e||!k.trigger||k.trigger.apply(d,c)!==!1)){if(!e&&!k.noBubble&&!n.isWindow(d)){for(i=k.delegateType||p,_.test(i+p)||(h=h.parentNode);h;h=h.parentNode)o.push(h),l=h;l===(d.ownerDocument||z)&&o.push(l.defaultView||l.parentWindow||a)}m=0;while((h=o[m++])&&!b.isPropagationStopped())b.type=m>1?i:k.bindType||p,f=(n._data(h,"events")||{})[b.type]&&n._data(h,"handle"),f&&f.apply(h,c),f=g&&h[g],f&&f.apply&&n.acceptData(h)&&(b.result=f.apply(h,c),b.result===!1&&b.preventDefault());if(b.type=p,!e&&!b.isDefaultPrevented()&&(!k._default||k._default.apply(o.pop(),c)===!1)&&n.acceptData(d)&&g&&d[p]&&!n.isWindow(d)){l=d[g],l&&(d[g]=null),n.event.triggered=p;try{d[p]()}catch(r){}n.event.triggered=void 0,l&&(d[g]=l)}return b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,e,f,g,h=[],i=d.call(arguments),j=(n._data(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,g=0;while((e=f.handlers[g++])&&!a.isImmediatePropagationStopped())(!a.namespace_re||a.namespace_re.test(e.namespace))&&(a.handleObj=e,a.data=e.data,c=((n.event.special[e.origType]||{}).handle||e.handler).apply(f.elem,i),void 0!==c&&(a.result=c)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&(!a.button||"click"!==a.type))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(e=[],f=0;h>f;f++)d=b[f],c=d.selector+" ",void 0===e[c]&&(e[c]=d.needsContext?n(c,this).index(i)>=0:n.find(c,this,null,[i]).length),e[c]&&e.push(d);e.length&&g.push({elem:i,handlers:e})}return h]","i"),ib=/^\s+/,jb=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,kb=/<([\w:]+)/,lb=/\s*$/g,sb={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:l.htmlSerialize?[0,"",""]:[1,"X
","
"]},tb=eb(z),ub=tb.appendChild(z.createElement("div"));sb.optgroup=sb.option,sb.tbody=sb.tfoot=sb.colgroup=sb.caption=sb.thead,sb.th=sb.td;function vb(a,b){var c,d,e=0,f=typeof a.getElementsByTagName!==L?a.getElementsByTagName(b||"*"):typeof a.querySelectorAll!==L?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||n.nodeName(d,b)?f.push(d):n.merge(f,vb(d,b));return void 0===b||b&&n.nodeName(a,b)?n.merge([a],f):f}function wb(a){X.test(a.type)&&(a.defaultChecked=a.checked)}function xb(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function yb(a){return a.type=(null!==n.find.attr(a,"type"))+"/"+a.type,a}function zb(a){var b=qb.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Ab(a,b){for(var c,d=0;null!=(c=a[d]);d++)n._data(c,"globalEval",!b||n._data(b[d],"globalEval"))}function Bb(a,b){if(1===b.nodeType&&n.hasData(a)){var c,d,e,f=n._data(a),g=n._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)n.event.add(b,c,h[c][d])}g.data&&(g.data=n.extend({},g.data))}}function Cb(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!l.noCloneEvent&&b[n.expando]){e=n._data(b);for(d in e.events)n.removeEvent(b,d,e.handle);b.removeAttribute(n.expando)}"script"===c&&b.text!==a.text?(yb(b).text=a.text,zb(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),l.html5Clone&&a.innerHTML&&!n.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&X.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:("input"===c||"textarea"===c)&&(b.defaultValue=a.defaultValue)}}n.extend({clone:function(a,b,c){var d,e,f,g,h,i=n.contains(a.ownerDocument,a);if(l.html5Clone||n.isXMLDoc(a)||!hb.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(ub.innerHTML=a.outerHTML,ub.removeChild(f=ub.firstChild)),!(l.noCloneEvent&&l.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(d=vb(f),h=vb(a),g=0;null!=(e=h[g]);++g)d[g]&&Cb(e,d[g]);if(b)if(c)for(h=h||vb(a),d=d||vb(f),g=0;null!=(e=h[g]);g++)Bb(e,d[g]);else Bb(a,f);return d=vb(f,"script"),d.length>0&&Ab(d,!i&&vb(a,"script")),d=h=e=null,f},buildFragment:function(a,b,c,d){for(var e,f,g,h,i,j,k,m=a.length,o=eb(b),p=[],q=0;m>q;q++)if(f=a[q],f||0===f)if("object"===n.type(f))n.merge(p,f.nodeType?[f]:f);else if(mb.test(f)){h=h||o.appendChild(b.createElement("div")),i=(kb.exec(f)||["",""])[1].toLowerCase(),k=sb[i]||sb._default,h.innerHTML=k[1]+f.replace(jb,"<$1>")+k[2],e=k[0];while(e--)h=h.lastChild;if(!l.leadingWhitespace&&ib.test(f)&&p.push(b.createTextNode(ib.exec(f)[0])),!l.tbody){f="table"!==i||lb.test(f)?""!==k[1]||lb.test(f)?0:h:h.firstChild,e=f&&f.childNodes.length;while(e--)n.nodeName(j=f.childNodes[e],"tbody")&&!j.childNodes.length&&f.removeChild(j)}n.merge(p,h.childNodes),h.textContent="";while(h.firstChild)h.removeChild(h.firstChild);h=o.lastChild}else p.push(b.createTextNode(f));h&&o.removeChild(h),l.appendChecked||n.grep(vb(p,"input"),wb),q=0;while(f=p[q++])if((!d||-1===n.inArray(f,d))&&(g=n.contains(f.ownerDocument,f),h=vb(o.appendChild(f),"script"),g&&Ab(h),c)){e=0;while(f=h[e++])pb.test(f.type||"")&&c.push(f)}return h=null,o},cleanData:function(a,b){for(var d,e,f,g,h=0,i=n.expando,j=n.cache,k=l.deleteExpando,m=n.event.special;null!=(d=a[h]);h++)if((b||n.acceptData(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)m[e]?n.event.remove(d,e):n.removeEvent(d,e,g.handle);j[f]&&(delete j[f],k?delete d[i]:typeof d.removeAttribute!==L?d.removeAttribute(i):d[i]=null,c.push(f))}}}),n.fn.extend({text:function(a){return W(this,function(a){return void 0===a?n.text(this):this.empty().append((this[0]&&this[0].ownerDocument||z).createTextNode(a))},null,a,arguments.length)},append:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=xb(this,a);b.appendChild(a)}})},prepend:function(){return this.domManip(arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=xb(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return this.domManip(arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},remove:function(a,b){for(var c,d=a?n.filter(a,this):this,e=0;null!=(c=d[e]);e++)b||1!==c.nodeType||n.cleanData(vb(c)),c.parentNode&&(b&&n.contains(c.ownerDocument,c)&&Ab(vb(c,"script")),c.parentNode.removeChild(c));return this},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&n.cleanData(vb(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&n.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return W(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(gb,""):void 0;if(!("string"!=typeof a||nb.test(a)||!l.htmlSerialize&&hb.test(a)||!l.leadingWhitespace&&ib.test(a)||sb[(kb.exec(a)||["",""])[1].toLowerCase()])){a=a.replace(jb,"<$1>");try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(vb(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=arguments[0];return this.domManip(arguments,function(b){a=this.parentNode,n.cleanData(vb(this)),a&&a.replaceChild(b,this)}),a&&(a.length||a.nodeType)?this:this.remove()},detach:function(a){return this.remove(a,!0)},domManip:function(a,b){a=e.apply([],a);var c,d,f,g,h,i,j=0,k=this.length,m=this,o=k-1,p=a[0],q=n.isFunction(p);if(q||k>1&&"string"==typeof p&&!l.checkClone&&ob.test(p))return this.each(function(c){var d=m.eq(c);q&&(a[0]=p.call(this,c,d.html())),d.domManip(a,b)});if(k&&(i=n.buildFragment(a,this[0].ownerDocument,!1,this),c=i.firstChild,1===i.childNodes.length&&(i=c),c)){for(g=n.map(vb(i,"script"),yb),f=g.length;k>j;j++)d=i,j!==o&&(d=n.clone(d,!0,!0),f&&n.merge(g,vb(d,"script"))),b.call(this[j],d,j);if(f)for(h=g[g.length-1].ownerDocument,n.map(g,zb),j=0;f>j;j++)d=g[j],pb.test(d.type||"")&&!n._data(d,"globalEval")&&n.contains(h,d)&&(d.src?n._evalUrl&&n._evalUrl(d.src):n.globalEval((d.text||d.textContent||d.innerHTML||"").replace(rb,"")));i=c=null}return this}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=0,e=[],g=n(a),h=g.length-1;h>=d;d++)c=d===h?this:this.clone(!0),n(g[d])[b](c),f.apply(e,c.get());return this.pushStack(e)}});var Db,Eb={};function Fb(b,c){var d=n(c.createElement(b)).appendTo(c.body),e=a.getDefaultComputedStyle?a.getDefaultComputedStyle(d[0]).display:n.css(d[0],"display");return d.detach(),e}function Gb(a){var b=z,c=Eb[a];return c||(c=Fb(a,b),"none"!==c&&c||(Db=(Db||n("