Processing a user in the queue now downloads all available replays (best effort)

nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt

@@ -13,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService
 import com.nisemoe.nise.konata.Replay
 import com.nisemoe.nise.konata.ReplaySetComparison
 import com.nisemoe.nise.konata.compareReplaySet
-import com.nisemoe.nise.osu.Mod
+import com.nisemoe.nise.osu.*
-import com.nisemoe.nise.osu.OsuApi
-import com.nisemoe.nise.osu.OsuApiModels
 import com.nisemoe.nise.service.CacheService
 import com.nisemoe.nise.service.CompressReplay
 import com.nisemoe.nise.service.UpdateUserQueueService
@@ -165,38 +163,86 @@ class ImportScores(
         }
 
         for(userId in queue) {
-            val topUserScores = this.osuApi.getTopUserScores(userId = userId)
+            val user = this.osuApi.getUserProfile(userId.toString())
-            val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
+
-            val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+            if (user == null) {
+                this.logger.error("Failed to fetch user from queue $userId")
+                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                continue;
+            }
+
+            var userScores = mutableListOf<OsuApiModels.Score>()
+
+            if (user.beatmap_playcounts_count != null) {
+                val mapsPlayed: MutableSet<Int> = mutableSetOf()
+
+                this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
+
+                for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
+                    val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
+                        ?: break
+
+                    mapsPlayed.addAll(maps.map { it.beatmap_id })
+
+                    this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+                var scoreProcessCount = 0
+                for (mapId in mapsPlayed) {
+                    val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
+                        ?: continue
+
+                    for (mapScore in scores.scores) {
+                        if (mapScore.replay && mapScore.id != null) {
+                            val beatmap = this.osuApi.getBeatmapFromId(mapId)
+                                ?: continue
+
+                            userScores.add(mapScore.copy(
+                                beatmap = beatmap.toScoreBeatmap(),
+                                beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
+                            ))
+                        }
+                    }
+
+                    this.logger.info(
+                        "Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
+                    )
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+            } else {
+                val topUserScores = this.osuApi.getTopUserScores(userId = userId)
+                val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
+                val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+
+                if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
+                    this.logger.error("Failed to fetch top scores for user with id = $userId")
+                    this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                    continue
+                }
+
+                userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
+
+                userScores = userScores
+                    .filter { it.beatmap != null && it.beatmapset != null }
+                    .distinctBy { it.best_id }
+                    .toMutableList()
+            }
 
             this.logger.info("Processing user with id = $userId")
-            this.logger.info("Top scores: ${topUserScores?.size}")
+            this.logger.info("User has ${userScores.size} total scores")
-            this.logger.info("Recent scores: ${recentUserScores?.size}")
-            this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
 
             Thread.sleep(SLEEP_AFTER_API_CALL)
 
-            if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
+            this.logger.info("Unique scores: ${userScores.size}")
-                this.logger.error("Failed to fetch top scores for user with id = $userId")
-                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
-                continue
-            }
-
-            val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
-                .filter { it.beatmap != null && it.beatmapset != null }
-                .distinctBy { it.best_id }
-
-            this.logger.info("Unique scores: ${allUserScores.size}")
 
             val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
-            if(!userExists) {
+            if (!userExists) {
-                val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
+                this.userService.insertApiUser(user)
-                if(apiUser != null) {
+                this.statistics.usersAddedToDatabase++
-                    this.userService.insertApiUser(apiUser)
-                    this.statistics.usersAddedToDatabase++
-                } else {
-                    this.logger.error("Failed to fetch user with id = $userId")
-                }
             }
 
             var current = 0
@@ -209,7 +255,7 @@ class ImportScores(
                 .limit(1)
                 .fetchOneInto(OffsetDateTime::class.java)
 
-            for(topScore in allUserScores) {
+            for(topScore in userScores) {
                 val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
                 if (!beatmapExists) {
                     val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@@ -264,7 +310,7 @@ class ImportScores(
                 // Update the database
                 dslContext.update(UPDATE_USER_QUEUE)
                     .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
-                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
+                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
                     .where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
                     .and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
                     .execute()
@@ -274,7 +320,7 @@ class ImportScores(
                     lastCompletedUpdate = lastCompletedUpdate,
                     canUpdate = false,
                     progressCurrent = current,
-                    progressTotal = allUserScores.size
+                    progressTotal = userScores.size
                 )
 
                 // Update the frontend
@@ -289,7 +335,7 @@ class ImportScores(
             }
 
             // Check for stolen replays.
-            val uniqueBeatmapIds = allUserScores
+            val uniqueBeatmapIds = userScores
                 .groupBy { it.beatmap!!.id }
 
             this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")