Processing a user in the queue now downloads all available replays (best effort)

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt

@@ -137,6 +137,19 @@ class OsuApi(
         }
     }
 
+    fun getBeatmapFromId(beatmapId: Int): OsuApiModels.Beatmap? {
+        val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId", emptyMap())
+        if (response == null) {
+            this.logger.info("Error loading beatmap $beatmapId")
+            return null
+        }
+
+        return when (response.statusCode()) {
+            200 -> serializer.decodeFromString<OsuApiModels.Beatmap>(response.body())
+            else -> null
+        }
+    }
+
     /**
      * Retrieves the replay data for a given score ID from the osu!api.
      * Efficiently cycles through the API keys to avoid rate limiting.
@@ -215,7 +228,7 @@ class OsuApi(
     }
 
     fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? {
-        val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", mapOf())
+        val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", emptyMap())
 
         if(response == null) {
             this.logger.info("Error getting scores on beatmap $beatmapId for user $userId")
@@ -249,7 +262,7 @@ class OsuApi(
     }
 
     fun checkIfUserBanned(userId: Long): Boolean? {
-        val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", mapOf())
+        val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", emptyMap())
         if(response == null) {
             this.logger.info("Error loading user with userId = $userId")
             return null

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt

@@ -205,6 +205,7 @@ class OsuApiModels {
     data class Beatmap(
         val beatmapset_id: Int,
         val difficulty_rating: Double?,
+        val checksum: String?,
         val id: Int,
         val version: String?,
         val beatmapset: BeatmapSet,
@@ -222,6 +223,7 @@ class OsuApiModels {
 
     @Serializable
     data class BeatmapSet(
+        val id: Int,
         val artist: String?,
         val creator: String?,
         val source: String?,

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt

@@ -0,0 +1,28 @@
+package com.nisemoe.nise.osu
+
+fun OsuApiModels.Beatmap.toScoreBeatmap(): OsuApiModels.ScoreBeatmap =
+    OsuApiModels.ScoreBeatmap(
+        id = this.id,
+        checksum = this.checksum,
+        difficulty_rating = this.difficulty_rating,
+        version = this.version,
+        max_combo = this.max_combo,
+        total_length = this.total_length,
+        bpm = this.bpm,
+        accuracy = this.accuracy,
+        ar = this.ar,
+        cs = this.cs,
+        drain = this.drain,
+        count_circles = this.count_circles,
+        count_sliders = this.count_sliders,
+        count_spinners = this.count_spinners,
+    )
+
+fun OsuApiModels.BeatmapSet.toScoreBeatmapSet(): OsuApiModels.ScoreBeatmapset =
+    OsuApiModels.ScoreBeatmapset(
+        id = this.id,
+        title = this.title,
+        artist = this.artist,
+        creator = this.creator,
+        source = this.source,
+    )

nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt

@@ -13,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService
 import com.nisemoe.nise.konata.Replay
 import com.nisemoe.nise.konata.ReplaySetComparison
 import com.nisemoe.nise.konata.compareReplaySet
-import com.nisemoe.nise.osu.Mod
+import com.nisemoe.nise.osu.*
-import com.nisemoe.nise.osu.OsuApi
-import com.nisemoe.nise.osu.OsuApiModels
 import com.nisemoe.nise.service.CacheService
 import com.nisemoe.nise.service.CompressReplay
 import com.nisemoe.nise.service.UpdateUserQueueService
@@ -165,38 +163,86 @@ class ImportScores(
         }
 
         for(userId in queue) {
-            val topUserScores = this.osuApi.getTopUserScores(userId = userId)
+            val user = this.osuApi.getUserProfile(userId.toString())
-            val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
+
-            val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+            if (user == null) {
+                this.logger.error("Failed to fetch user from queue $userId")
+                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                continue;
+            }
+
+            var userScores = mutableListOf<OsuApiModels.Score>()
+
+            if (user.beatmap_playcounts_count != null) {
+                val mapsPlayed: MutableSet<Int> = mutableSetOf()
+
+                this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
+
+                for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
+                    val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
+                        ?: break
+
+                    mapsPlayed.addAll(maps.map { it.beatmap_id })
+
+                    this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+                var scoreProcessCount = 0
+                for (mapId in mapsPlayed) {
+                    val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
+                        ?: continue
+
+                    for (mapScore in scores.scores) {
+                        if (mapScore.replay && mapScore.id != null) {
+                            val beatmap = this.osuApi.getBeatmapFromId(mapId)
+                                ?: continue
+
+                            userScores.add(mapScore.copy(
+                                beatmap = beatmap.toScoreBeatmap(),
+                                beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
+                            ))
+                        }
+                    }
+
+                    this.logger.info(
+                        "Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
+                    )
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+            } else {
+                val topUserScores = this.osuApi.getTopUserScores(userId = userId)
+                val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
+                val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+
+                if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
+                    this.logger.error("Failed to fetch top scores for user with id = $userId")
+                    this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                    continue
+                }
+
+                userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
+
+                userScores = userScores
+                    .filter { it.beatmap != null && it.beatmapset != null }
+                    .distinctBy { it.best_id }
+                    .toMutableList()
+            }
 
             this.logger.info("Processing user with id = $userId")
-            this.logger.info("Top scores: ${topUserScores?.size}")
+            this.logger.info("User has ${userScores.size} total scores")
-            this.logger.info("Recent scores: ${recentUserScores?.size}")
-            this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
 
             Thread.sleep(SLEEP_AFTER_API_CALL)
 
-            if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
+            this.logger.info("Unique scores: ${userScores.size}")
-                this.logger.error("Failed to fetch top scores for user with id = $userId")
-                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
-                continue
-            }
-
-            val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
-                .filter { it.beatmap != null && it.beatmapset != null }
-                .distinctBy { it.best_id }
-
-            this.logger.info("Unique scores: ${allUserScores.size}")
 
             val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
-            if(!userExists) {
+            if (!userExists) {
-                val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
+                this.userService.insertApiUser(user)
-                if(apiUser != null) {
+                this.statistics.usersAddedToDatabase++
-                    this.userService.insertApiUser(apiUser)
-                    this.statistics.usersAddedToDatabase++
-                } else {
-                    this.logger.error("Failed to fetch user with id = $userId")
-                }
             }
 
             var current = 0
@@ -209,7 +255,7 @@ class ImportScores(
                 .limit(1)
                 .fetchOneInto(OffsetDateTime::class.java)
 
-            for(topScore in allUserScores) {
+            for(topScore in userScores) {
                 val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
                 if (!beatmapExists) {
                     val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@@ -264,7 +310,7 @@ class ImportScores(
                 // Update the database
                 dslContext.update(UPDATE_USER_QUEUE)
                     .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
-                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
+                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
                     .where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
                     .and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
                     .execute()
@@ -274,7 +320,7 @@ class ImportScores(
                     lastCompletedUpdate = lastCompletedUpdate,
                     canUpdate = false,
                     progressCurrent = current,
-                    progressTotal = allUserScores.size
+                    progressTotal = userScores.size
                 )
 
                 // Update the frontend
@@ -289,7 +335,7 @@ class ImportScores(
             }
 
             // Check for stolen replays.
-            val uniqueBeatmapIds = allUserScores
+            val uniqueBeatmapIds = userScores
                 .groupBy { it.beatmap!!.id }
 
             this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")