Skip to content

Commit 2c80c7e

Browse files
ornicarornicar
committed
Merge branch 'master' into tournament-restrict-team
* master: fix basic JS syntax inc assets version Manually apply translations better cp round refactor make GC work with boost implement relation API endpoints - closes lichess-org#4398 fix dup UGC add support for yet another DGT broadcast format GC tweak lilaify while reviewing note deletion type alias User note deletion, closes lichess-org#4371
2 parents feb0870 + 9155ca8 commit 2c80c7e

File tree

26 files changed

+271
-39
lines changed

26 files changed

+271
-39
lines changed

‎app/controllers/Relation.scala

+21
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
package controllers
22

3+
import play.api.libs.iteratee._
34
import play.api.libs.json.Json
5+
import play.api.mvc._
46

57
import lila.api.Context
68
import lila.app._
79
import lila.common.paginator.{ Paginator, AdapterLike, PaginatorJson }
10+
import lila.common.{ HTTPRequest, MaxPerSecond }
811
import lila.relation.Related
12+
import lila.relation.RelationStream._
913
import lila.user.{ User => UserModel, UserRepo }
1014
import views._
1115

@@ -76,6 +80,23 @@ object Relation extends LilaController {
7680
}
7781
}
7882

83+
def apiFollowing(name: String) = apiRelation(name, Direction.Following)
84+
85+
def apiFollowers(name: String) = apiRelation(name, Direction.Followers)
86+
87+
private def apiRelation(name: String, direction: Direction) = Action.async { req =>
88+
UserRepo.named(name) flatMap {
89+
_ ?? { user =>
90+
import Api.limitedDefault
91+
Api.GlobalLinearLimitPerIP(HTTPRequest lastRemoteAddress req) {
92+
Api.jsonStream {
93+
env.stream.follow(user, direction, MaxPerSecond(20)) &> Enumeratee.map(Env.api.userApi.one)
94+
} |> fuccess
95+
}
96+
}
97+
}
98+
}
99+
79100
private def jsonRelatedPaginator(pag: Paginator[Related]) = {
80101
import lila.user.JsonView.nameWrites
81102
import lila.relation.JsonView.relatedWrites

‎app/controllers/User.scala

+8
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,14 @@ object User extends LilaController {
283283
}
284284
}
285285

286+
def deleteNote(id: String) = Auth { implicit ctx => me =>
287+
OptionFuResult(env.noteApi.byId(id)) { note =>
288+
note.isFrom(me) ?? {
289+
env.noteApi.delete(note._id) inject Redirect(routes.User.show(note.to).url + "?note")
290+
}
291+
}
292+
}
293+
286294
def opponents = Auth { implicit ctx => me =>
287295
for {
288296
ops <- Env.game.bestOpponents(me.id)

‎app/views/user/show/header.scala.html

+9-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,15 @@ <h1 class="lichess_title user_link @if(isOnline(u.id)){online}else{offline}">
8383
}
8484
@notes.map { note =>
8585
<div>
86-
<p class="meta">@userIdLink(note.from.some)<br />@momentFromNow(note.date)</p>
86+
<p class="meta">
87+
@userIdLink(note.from.some)<br />@momentFromNow(note.date)
88+
@if(ctx.me.exists(note.isFrom)) {
89+
<br />
90+
<form action="@routes.User.deleteNote(note._id)" method="post">
91+
<button type="submit" class="thin confirm button text" style="float:right" data-icon="q">Delete</button>
92+
</form>
93+
}
94+
</p>
8795
<p class="text">@richText(note.text)</p>
8896
</div>
8997
}

‎build.sbt

+1-1
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ lazy val plan = module("plan", Seq(common, user, notifyModule)).settings(
329329
)
330330

331331
lazy val relation = module("relation", Seq(common, db, memo, hub, user, game, pref)).settings(
332-
libraryDependencies ++= provided(play.api, reactivemongo.driver)
332+
libraryDependencies ++= provided(play.api, reactivemongo.driver, reactivemongo.iteratees)
333333
)
334334

335335
lazy val pref = module("pref", Seq(common, db, user)).settings(

‎conf/base.conf

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ net {
1111
ip = "5.196.91.160"
1212
asset {
1313
domain = "lichess-assets.local"
14-
version = 2060
14+
version = 2075
1515
}
1616
email = "[email protected]"
1717
crawlable = false

‎conf/routes

+3
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ GET /@/:username/tournaments/:path controllers.UserTournament.path(username:
5050
# User
5151
GET /@/:username/mod controllers.User.mod(username: String)
5252
POST /@/:username/note controllers.User.writeNote(username: String)
53+
POST /note/delete/:id controllers.User.deleteNote(id: String)
5354
GET /@/:username/mini controllers.User.showMini(username: String)
5455
GET /@/:username/tv controllers.User.tv(username: String)
5556
GET /@/:username/studyTv controllers.User.studyTv(username: String)
@@ -493,6 +494,8 @@ GET /api controllers.Api.index
493494
POST /api/users controllers.Api.usersByIds
494495
GET /api/user/:name controllers.Api.user(name: String)
495496
GET /api/user/:name/activity controllers.Api.activity(name: String)
497+
GET /api/user/:name/following controllers.Relation.apiFollowing(name: String)
498+
GET /api/user/:name/followers controllers.Relation.apiFollowers(name: String)
496499
GET /api/game/:id controllers.Api.game(id: String)
497500
GET /api/games/team/:teamId controllers.Api.gamesVsTeam(teamId: String)
498501
GET /api/tournament controllers.Api.currentTournaments

‎modules/chat/src/main/ChatPanic.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ final class ChatPanic {
2424

2525
def start = {
2626
logger.warn("Chat Panic enabled")
27-
until = DateTime.now.plusMinutes(60).some
27+
until = DateTime.now.plusMinutes(180).some
2828
}
2929

3030
def stop = {

‎modules/hub/src/main/actorApi.scala

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ package report {
5151

5252
package security {
5353
case class GarbageCollect(userId: String, ipBan: Boolean)
54+
case class GCImmediateSb(userId: String)
5455
case class CloseAccount(userId: String)
5556
}
5657

‎modules/memo/src/main/ExpireSetMemo.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ final class ExpireSetMemo(ttl: Duration) {
99
.expireAfterWrite(ttl)
1010
.build[String, Boolean]
1111

12-
private def isNotNull[A](a: A) = a != null
12+
@inline private def isNotNull[A](a: A) = a != null
1313

1414
def get(key: String): Boolean = isNotNull(cache.underlying getIfPresent key)
1515

‎modules/mod/src/main/Env.scala

+6
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,12 @@ final class Env(
123123
if (game.status == chess.Status.Cheat)
124124
game.loserUserId foreach { logApi.cheatDetected(_, game.id) }
125125
case lila.hub.actorApi.mod.ChatTimeout(mod, user, reason) => logApi.chatTimeout(mod, user, reason)
126+
case lila.hub.actorApi.security.GCImmediateSb(userId) =>
127+
reportApi getSuspect userId flatten s"No such suspect $userId" flatMap { sus =>
128+
reportApi.getLichessMod map { mod =>
129+
api.setTroll(mod, sus, true)
130+
}
131+
}
126132
case lila.hub.actorApi.security.GarbageCollect(userId, ipBan) =>
127133
reportApi getSuspect userId flatten s"No such suspect $userId" flatMap { sus =>
128134
api.garbageCollect(sus, ipBan) >> publicChat.delete(sus)

‎modules/mod/src/main/ModApi.scala

-1
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,6 @@ final class ModApi(
9393
def garbageCollect(sus: Suspect, ipBan: Boolean): Funit = for {
9494
mod <- reportApi.getLichessMod
9595
_ <- setEngine(mod, sus, true)
96-
_ <- setTroll(mod, sus, true)
9796
_ <- ipBan ?? setBan(mod, sus, true)
9897
} yield logApi.garbageCollect(mod, sus)
9998

‎modules/puzzle/src/main/PuzzleBatch.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ private[puzzle] final class PuzzleBatch(
88
puzzleColl: Coll,
99
api: PuzzleApi,
1010
finisher: Finisher,
11-
puzzleIdMin: Int
11+
puzzleIdMin: PuzzleId
1212
) {
1313

1414
def solve(originalUser: User, data: PuzzleBatch.SolveData): Funit = for {

‎modules/relation/src/main/Env.scala

+2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ final class Env(
3939
maxBlock = MaxBlock
4040
)
4141

42+
lazy val stream = new RelationStream(coll = coll)(system)
43+
4244
val online = new OnlineDoing(
4345
api,
4446
lightUser = lightUserApi.sync,

‎modules/relation/src/main/RelationStream.scala

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package lila.relation
2+
3+
import play.api.libs.iteratee._
4+
import reactivemongo.api.ReadPreference
5+
import reactivemongo.play.iteratees.cursorProducer
6+
import scala.concurrent.duration._
7+
8+
import lila.common.MaxPerSecond
9+
import lila.db.dsl._
10+
import lila.user.{ User, UserRepo }
11+
12+
final class RelationStream(coll: Coll)(implicit system: akka.actor.ActorSystem) {
13+
14+
import RelationStream._
15+
16+
def follow(user: User, direction: Direction, perSecond: MaxPerSecond): Enumerator[User] = {
17+
val field = direction match {
18+
case Direction.Following => "u2"
19+
case Direction.Followers => "u1"
20+
}
21+
val projection = $doc(field -> true, "_id" -> false)
22+
val query = direction match {
23+
case Direction.Following => coll.find($doc("u1" -> user.id, "r" -> Follow), projection)
24+
case Direction.Followers => coll.find($doc("u2" -> user.id, "r" -> Follow), projection)
25+
}
26+
query.copy(options = query.options.batchSize(perSecond.value))
27+
.cursor[Bdoc](readPreference = ReadPreference.secondaryPreferred)
28+
.bulkEnumerator() &>
29+
lila.common.Iteratee.delay(1 second) &>
30+
Enumeratee.mapM { docs =>
31+
UserRepo usersFromSecondary docs.toSeq.flatMap(_.getAs[User.ID](field))
32+
} &>
33+
Enumeratee.mapConcat(_.toSeq)
34+
}
35+
}
36+
37+
object RelationStream {
38+
39+
sealed trait Direction
40+
object Direction {
41+
case object Following extends Direction
42+
case object Followers extends Direction
43+
}
44+
}

‎modules/relay/src/main/RelayFetch.scala

+61-18
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,14 @@ package lila.relay
22

33
import akka.actor._
44
import org.joda.time.DateTime
5-
import play.api.libs.ws.WS
5+
import play.api.libs.json._
6+
import play.api.libs.ws.{ WS, WSResponse }
67
import play.api.Play.current
78
import scala.concurrent.duration._
89

910
import lila.base.LilaException
10-
import lila.tree.Node.Comments
1111
import lila.study.MultiPgn
12+
import lila.tree.Node.Comments
1213

1314
private final class RelayFetch(
1415
sync: RelaySync,
@@ -130,7 +131,10 @@ private object RelayFetch {
130131

131132
private def doFetch(upstream: Upstream, max: Int): Fu[RelayGames] = (upstream match {
132133
case Upstream.DgtOneFile(file) => dgtOneFile(file, max)
133-
case Upstream.DgtManyFiles(dir) => dgtManyFiles(dir, max)
134+
case Upstream.DgtManyFiles(dir) =>
135+
dgtManyFiles(dir, max, DgtMany.RoundPgn) recoverWith {
136+
case _: lila.base.LilaException => dgtManyFiles(dir, max, DgtMany.Indexjson)
137+
}
134138
}) flatMap multiPgnToGames.apply
135139

136140
private def dgtOneFile(file: String, max: Int): Fu[MultiPgn] =
@@ -139,29 +143,68 @@ private object RelayFetch {
139143
case res => fufail(s"[${res.status}]")
140144
}
141145

142-
import play.api.libs.json._
146+
private object DgtJson {
147+
case class PairingPlayer(fname: Option[String], mname: Option[String], lname: Option[String], title: Option[String]) {
148+
def fullName = some {
149+
List(fname, mname, lname).flatten mkString " "
150+
}.filter(_.nonEmpty)
151+
}
152+
case class RoundJsonPairing(white: PairingPlayer, black: PairingPlayer, result: String) {
153+
import chess.format.pgn._
154+
def tags = Tags(List(
155+
white.fullName map { v => Tag(_.White, v) },
156+
white.title map { v => Tag(_.WhiteTitle, v) },
157+
black.fullName map { v => Tag(_.Black, v) },
158+
black.title map { v => Tag(_.BlackTitle, v) },
159+
Tag(_.Result, result).some
160+
).flatten)
161+
}
162+
case class RoundJson(pairings: List[RoundJsonPairing])
163+
implicit val pairingPlayerReads = Json.reads[PairingPlayer]
164+
implicit val roundPairingReads = Json.reads[RoundJsonPairing]
165+
implicit val roundReads = Json.reads[RoundJson]
143166

144-
private case class RoundJsonPairing(live: Boolean)
145-
private case class RoundJson(pairings: List[RoundJsonPairing])
146-
private implicit val roundPairingReads = Json.reads[RoundJsonPairing]
147-
private implicit val roundReads = Json.reads[RoundJson]
167+
case class GameJson(moves: List[String], result: Option[String])
168+
implicit val gameReads = Json.reads[GameJson]
169+
}
170+
import DgtJson._
171+
172+
private sealed abstract class DgtMany(val indexFile: String, val gameFile: Int => String, val toPgn: (WSResponse, RoundJsonPairing) => String)
173+
private object DgtMany {
174+
case object RoundPgn extends DgtMany("round.json", n => s"game-$n.pgn", (r, _) => r.body)
175+
case object Indexjson extends DgtMany("index.json", n => s"game-$n.json", {
176+
case (res, pairing) => res.json.validate[GameJson] match {
177+
case JsSuccess(game, _) =>
178+
val moves = game.moves.map(_ split ' ') map { move =>
179+
chess.format.pgn.Move(
180+
san = ~move.headOption,
181+
secondsLeft = move.lift(1).map(_.takeWhile(_.isDigit)) flatMap parseIntOption
182+
)
183+
} mkString " "
184+
s"${pairing.tags}\n\n$moves"
185+
case JsError(err) => ""
186+
}
187+
})
188+
}
148189

149-
private def dgtManyFiles(dir: String, max: Int): Fu[MultiPgn] = {
150-
val roundUrl = s"$dir/round.json"
151-
httpGet(roundUrl) flatMap {
190+
private def dgtManyFiles(dir: String, max: Int, format: DgtMany): Fu[MultiPgn] = {
191+
val indexFile = s"$dir/${format.indexFile}"
192+
httpGet(indexFile) flatMap {
152193
case res if res.status == 200 => roundReads reads res.json match {
153194
case JsError(err) => fufail(err.toString)
154-
case JsSuccess(round, _) => (1 to round.pairings.size.atMost(max)).map { number =>
155-
val gameUrl = s"$dir/game-$number.pgn"
156-
httpGet(gameUrl).flatMap {
157-
case res if res.status == 200 => fuccess(number -> res.body)
158-
case res => fufail(s"[${res.status}] game-$number.pgn")
159-
}
195+
case JsSuccess(round, _) => round.pairings.zipWithIndex.map {
196+
case (pairing, i) =>
197+
val number = i + 1
198+
val gameUrl = s"$dir/${format.gameFile(number)}"
199+
httpGet(gameUrl).flatMap {
200+
case res if res.status == 200 => fuccess(number -> format.toPgn(res, pairing))
201+
case res => fufail(s"[${res.status}] $gameUrl")
202+
}
160203
}.sequenceFu map { results =>
161204
MultiPgn(results.sortBy(_._1).map(_._2).toList)
162205
}
163206
}
164-
case res => fufail(s"[${res.status}] round.json")
207+
case res => fufail(s"[${res.status}] $indexFile")
165208
}
166209
}
167210

‎modules/security/src/main/GarbageCollector.scala

+18-4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ final class GarbageCollector(
1515
system: akka.actor.ActorSystem
1616
) {
1717

18+
private val done = new lila.memo.ExpireSetMemo(10 minutes)
19+
1820
// User just signed up and doesn't have security data yet, so wait a bit
1921
def delay(user: User, ip: IpAddress, email: EmailAddress): Unit =
2022
if (user.createdAt.isAfter(DateTime.now minusDays 3)) {
@@ -34,7 +36,10 @@ final class GarbageCollector(
3436
val ipBan = spy.usersSharingIp.forall { u =>
3537
isBadAccount(u) || !u.seenAt.exists(DateTime.now.minusMonths(2).isBefore)
3638
}
37-
collect(user, email, others, ipBan)
39+
if (!done.get(user.id)) {
40+
collect(user, email, others, ipBan)
41+
done put user.id
42+
}
3843
}
3944
}
4045
}
@@ -51,7 +56,7 @@ final class GarbageCollector(
5156
(others.size > 1 && others.forall(isBadAccount) && others.headOption.exists(_.disabled)) option others
5257
}
5358

54-
private def isBadAccount(user: User) = user.troll || user.engine
59+
private def isBadAccount(user: User) = user.lameOrTroll
5560

5661
private def collect(user: User, email: EmailAddress, others: List[User], ipBan: Boolean): Funit = {
5762
val armed = isArmed()
@@ -60,12 +65,21 @@ final class GarbageCollector(
6065
val message = s"Will dispose of @${user.username} in $wait. Email: $email. Prev users: $othersStr${!armed ?? " [SIMULATION]"}"
6166
logger.branch("GarbageCollector").info(message)
6267
slack.garbageCollector(message) >>- {
63-
if (armed) system.scheduler.scheduleOnce(wait) {
64-
doCollect(user, ipBan)
68+
if (armed) {
69+
doInitialSb(user)
70+
system.scheduler.scheduleOnce(wait) {
71+
doCollect(user, ipBan)
72+
}
6573
}
6674
}
6775
}
6876

77+
private def doInitialSb(user: User): Unit =
78+
system.lilaBus.publish(
79+
lila.hub.actorApi.security.GCImmediateSb(user.id),
80+
'garbageCollect
81+
)
82+
6983
private def doCollect(user: User, ipBan: Boolean): Unit =
7084
system.lilaBus.publish(
7185
lila.hub.actorApi.security.GarbageCollect(user.id, ipBan),

0 commit comments

Comments
 (0)