Merge branch 'pixel_tracker'

commons/src/main/scala/io/prediction/commons/appdata/Items.scala

@@ -20,7 +20,7 @@ case class Item(
   id: String,
   appid: Int,
   ct: DateTime,
-  itypes: List[String],
+  itypes: Seq[String],
   starttime: Option[DateTime],
   endtime: Option[DateTime],
   price: Option[Double],
@@ -38,8 +38,11 @@ trait Items {
   /** Get an item by ID. */
   def get(appid: Int, id: String): Option[Item]
 
+  /** Get items by IDs. */
+  def getByIds(appid: Int, ids: Seq[String]): Seq[Item]
+
   /** Get items by IDs sorted by their start time in descending order. */
-  def getRecentByIds(appid: Int, ids: List[String]): List[Item]
+  def getRecentByIds(appid: Int, ids: Seq[String]): Seq[Item]
 
   /** Update an item. */
   def update(item: Item): Unit

commons/src/main/scala/io/prediction/commons/appdata/U2IActions.scala

@@ -49,7 +49,7 @@ trait U2IActions {
   def getAllByAppid(appid: Int): Iterator[U2IAction]
 
   /** Gets all user-to-item actions by App ID, User ID, and Item IDs. */
-  def getAllByAppidAndUidAndIids(appid: Int, uid: String, iids: List[String]): Iterator[U2IAction]
+  def getAllByAppidAndUidAndIids(appid: Int, uid: String, iids: Seq[String]): Iterator[U2IAction]
 
   /** Delete all user-to-item actions by App ID */
   def deleteByAppid(appid: Int): Unit

commons/src/main/scala/io/prediction/commons/appdata/mongodb/MongoItems.scala

@@ -36,7 +36,11 @@ class MongoItems(db: MongoDB) extends Items {
     itemColl.findOne(MongoDBObject("_id" -> idWithAppid(appid, id))) map { dbObjToItem(_) }
   }
 
-  def getRecentByIds(appid: Int, ids: List[String]) = {
+  def getByIds(appid: Int, ids: Seq[String]) = {
+    itemColl.find(MongoDBObject("_id" -> MongoDBObject("$in" -> ids.map(idWithAppid(appid, _))))).toList map { dbObjToItem(_) }
+  }
+
+  def getRecentByIds(appid: Int, ids: Seq[String]) = {
     itemColl.find(MongoDBObject("_id" -> MongoDBObject("$in" -> ids.map(idWithAppid(appid, _))))).sort(starttimeIndex).toList map { dbObjToItem(_) }
   }
 

commons/src/main/scala/io/prediction/commons/appdata/mongodb/MongoU2IActions.scala

@@ -29,7 +29,7 @@ class MongoU2IActions(db: MongoDB) extends U2IActions {
 
   def getAllByAppid(appid: Int) = new MongoU2IActionIterator(u2iActionColl.find(MongoDBObject("appid" -> appid)))
 
-  def getAllByAppidAndUidAndIids(appid: Int, uid: String, iids: List[String]) = new MongoU2IActionIterator(
+  def getAllByAppidAndUidAndIids(appid: Int, uid: String, iids: Seq[String]) = new MongoU2IActionIterator(
     u2iActionColl.find(MongoDBObject("appid" -> appid, "uid" -> idWithAppid(appid, uid), "iid" -> MongoDBObject("$in" -> iids.map(idWithAppid(appid, _)))))
   )
 

commons/src/test/scala/io/prediction/commons/appdata/ItemsSpec.scala

@@ -19,6 +19,7 @@ class ItemsSpec extends Specification { def is =
 
   def items(items: Items) = {                                                 t ^
     "inserting and getting an item"                                           ! insert(items) ^
+    "getting items by IDs"                                                    ! getByIds(items) ^
     "getting items by IDs sorted by start time"                               ! getRecentByIds(items) ^
     "updating an item"                                                        ! update(items) ^
     "deleting an item"                                                        ! delete(items) ^
@@ -50,6 +51,63 @@ class ItemsSpec extends Specification { def is =
     items.get(appid, id) must beSome(item)
   }
 
+  def getByIds(items: Items) = {
+    val id = "getByIds"
+    val appid = 4
+    val someItems = List(Item(
+      id         = id + "foo",
+      appid      = appid,
+      ct         = DateTime.now,
+      itypes     = List("fresh", "meat"),
+      starttime  = Some(DateTime.now.hour(14).minute(13)),
+      endtime    = None,
+      price      = Some(49.394),
+      profit     = None,
+      latlng     = Some((47.8948, -29.79783)),
+      inactive   = None,
+      attributes = Some(Map("foo" -> "bar"))
+    ), Item(
+      id         = id + "bar",
+      appid      = appid,
+      ct         = DateTime.now,
+      itypes     = List("fresh", "meat"),
+      starttime  = Some(DateTime.now.hour(23).minute(13)),
+      endtime    = None,
+      price      = Some(49.394),
+      profit     = None,
+      latlng     = Some((47.8948, -29.79783)),
+      inactive   = None,
+      attributes = Some(Map("foo" -> "bar"))
+    ), Item(
+      id         = id + "baz",
+      appid      = appid,
+      ct         = DateTime.now,
+      itypes     = List("fresh", "meat"),
+      starttime  = Some(DateTime.now.hour(17).minute(13)),
+      endtime    = None,
+      price      = Some(49.394),
+      profit     = None,
+      latlng     = Some((47.8948, -29.79783)),
+      inactive   = None,
+      attributes = Some(Map("foo" -> "bar"))
+    ), Item(
+      id         = id + "pub",
+      appid      = appid,
+      ct         = DateTime.now,
+      itypes     = List("fresh", "meat"),
+      starttime  = Some(DateTime.now.hour(3).minute(13)),
+      endtime    = None,
+      price      = Some(49.394),
+      profit     = None,
+      latlng     = Some((47.8948, -29.79783)),
+      inactive   = None,
+      attributes = Some(Map("foo" -> "bar"))
+    ))
+    someItems foreach { items.insert(_) }
+    val setOfItems = items.getByIds(appid, List(id + "pub", id + "bar", id + "baz")).toSet
+    setOfItems.contains(someItems(1)) and setOfItems.contains(someItems(2)) and setOfItems.contains(someItems(3))
+  }
+
   def getRecentByIds(items: Items) = {
     val id = "getRecentByIds"
     val appid = 3

output/src/main/scala/io/prediction/output/itemrec/ItemRecAlgoOutput.scala

@@ -39,21 +39,21 @@ object ItemRecAlgoOutput {
 
       while (outputBuffer.length < finalN && !stopMore) {
         val moreItemRecScores = more(uid, finalN, itypes, after)
-        val moreIids = moreItemRecScores.map(_.iid).toList
+        val moreIids = moreItemRecScores.map(_.iid).toSeq
 
         /** Stop the loop if no more scores can be found. */
         if (moreItemRecScores.length == 0)
           stopMore = true
         else {
           val seenItems = u2iActions.getAllByAppidAndUidAndIids(app.id, uid, moreIids)
-          outputBuffer ++= (moreIids filterNot (seenItems.toList.map(_.iid).contains))
+          outputBuffer ++= (moreIids filterNot (seenItems.toSeq.map(_.iid).contains))
           after = Some(moreItemRecScores.last)
         }
       }
     } else outputBuffer ++= more(uid, finalN, itypes, None) map { _.iid }
 
     /** At this point "output" is guaranteed to have n*(s+1) items (seen or unseen) unless model data is exhausted. */
-    val output = outputBuffer.toList.take(finalN)
+    val output = outputBuffer.toSeq.take(finalN)
 
     /** Serendipity output. */
     val serendipityOutput = serendipity map { s =>

process/commons/hadoop/scalding/build.sbt

@@ -6,6 +6,8 @@ version := "0.4-SNAPSHOT"
 
 scalaVersion := "2.9.2"
 
+javacOptions ++= Seq("-source", "1.6", "-target", "1.6")
+
 parallelExecution in Test := false
 
 libraryDependencies += "org.apache.hadoop" % "hadoop-core" % "1.0.3"

servers/api/app/io/prediction/api/API.scala

@@ -13,6 +13,8 @@ import play.api.data.format.Formats._
 import play.api.data.validation._
 import play.api.i18n._
 import play.api.libs.json._
+import play.api.libs.iteratee.Enumerator
+import play.api.Play.current
 
 //import com.codahale.jerkson.Json._
 import org.joda.time._
@@ -95,6 +97,12 @@ object API extends Controller {
   def FormattedResponse(format: String)(r: APIResponse) = {
     format match {
       case "json" => (new Status(r.status)(Json.stringify(Json.toJson(r)))).as(JSON)
+      case "png" => Play.resourceAsStream("public/images/spacer.png") map { stream =>
+        val fileContent: Enumerator[Array[Byte]] = Enumerator.fromStream(stream)
+        SimpleResult(
+          header = ResponseHeader(200),
+          body = fileContent)
+      } getOrElse notFound
       case _ => notFound
     }
   }
@@ -230,7 +238,7 @@ object API extends Controller {
               ct = DateTime.now,
               latlng = latlng map { parseLatlng(_) },
               inactive = inactive,
-              attributes = Some(attributes)
+              attributes = if (attributes.isEmpty) None else Some(attributes)
             ))
             APIMessageResponse(CREATED, Map("message" -> "User created."))
           }
@@ -488,11 +496,12 @@ object API extends Controller {
         "itypes" -> optional(text),
         "latlng" -> optional(latlng),
         "within" -> optional(numeric),
-        "unit" -> optional(text)
+        "unit" -> optional(text),
+        "attributes" -> optional(text)
       )).bindFromRequest.fold(
         f => bindFailed(f.errors),
         t => {
-          val (appkey, uid, n, itypes, latlng, within, unit) = t
+          val (appkey, uid, n, itypes, latlng, within, unit, attributes) = t
           AuthenticatedApp(appkey) { implicit app =>
             ValidEngine(enginename) { implicit engine =>
               try {
@@ -501,10 +510,26 @@ object API extends Controller {
                   n = n,
                   itypes = itypes map { _.split(",").toList }
                 )
-                if (res.length > 0)
-                  APIMessageResponse(OK, Map("iids" -> res))
-                else
+                if (res.length > 0) {
+                  val attributesToGet = attributes map { _.split(",").toSeq } getOrElse Seq()
+
+                  if (attributesToGet.length > 0) {
+                    val attributedItems = items.getByIds(app.id, res) map { i => (i.id, i) } toMap
+                    val ar = attributesToGet map { atg =>
+                      Map(atg -> res.map(ri =>
+                        attributedItems(ri).attributes map { attribs =>
+                          attribs.get(atg) getOrElse null
+                        } getOrElse null
+                      ))
+                    }
+
+                    APIMessageResponse(OK, Map("iids" -> res) ++ ar.reduceLeft((a, b) => a ++ b))
+                  } else {
+                    APIMessageResponse(OK, Map("iids" -> res))
+                  }
+                } else {
                   APIMessageResponse(NOT_FOUND, Map("message" -> "Cannot find recommendation for user."))
+                }
               } catch {
                 case e: Exception =>
                   APIMessageResponse(INTERNAL_SERVER_ERROR, Map("message" -> e.getMessage()))

servers/api/conf/routes

@@ -18,4 +18,13 @@ POST    /actions/u2i/dislike.:format              io.prediction.api.API.userToIt
 POST    /actions/u2i/view.:format                 io.prediction.api.API.userToItemView(format)
 POST    /actions/u2i/conversion.:format           io.prediction.api.API.userToItemConversion(format)
 
+# For pixel tracking
+GET     /p/users.:format                          io.prediction.api.API.createUser(format)
+GET     /p/items.:format                          io.prediction.api.API.createItem(format)
+GET     /p/actions/u2i/rate.:format               io.prediction.api.API.userToItemRate(format)
+GET     /p/actions/u2i/like.:format               io.prediction.api.API.userToItemLike(format)
+GET     /p/actions/u2i/dislike.:format            io.prediction.api.API.userToItemDislike(format)
+GET     /p/actions/u2i/view.:format               io.prediction.api.API.userToItemView(format)
+GET     /p/actions/u2i/conversion.:format         io.prediction.api.API.userToItemConversion(format)
+
 GET     /engines/itemrec/:engine/topn.:format     io.prediction.api.API.itemRecTopN(format, engine)