diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index 1c60490..31e2d84 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -13,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService import com.nisemoe.nise.konata.Replay import com.nisemoe.nise.konata.ReplaySetComparison import com.nisemoe.nise.konata.compareReplaySet -import com.nisemoe.nise.osu.Mod -import com.nisemoe.nise.osu.OsuApi -import com.nisemoe.nise.osu.OsuApiModels +import com.nisemoe.nise.osu.* import com.nisemoe.nise.service.CacheService import com.nisemoe.nise.service.CompressReplay import com.nisemoe.nise.service.UpdateUserQueueService @@ -165,38 +163,86 @@ class ImportScores( } for(userId in queue) { - val topUserScores = this.osuApi.getTopUserScores(userId = userId) - val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent") - val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts") + val user = this.osuApi.getUserProfile(userId.toString()) + + if (user == null) { + this.logger.error("Failed to fetch user from queue $userId") + this.updateUserQueueService.setUserAsProcessed(userId, failed = true) + continue; + } + + var userScores = mutableListOf() + + if (user.beatmap_playcounts_count != null) { + val mapsPlayed: MutableSet = mutableSetOf() + + this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays") + + for (page in 1..(user.beatmap_playcounts_count / 50) + 1) { + val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page) + ?: break + + mapsPlayed.addAll(maps.map { it.beatmap_id }) + + this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}") + + Thread.sleep(SLEEP_AFTER_API_CALL) + } + + var scoreProcessCount = 0 + for (mapId in mapsPlayed) { + val scores = this.osuApi.getUserBeatmapScores(userId, mapId) + ?: continue + + for (mapScore in scores.scores) { + if (mapScore.replay && mapScore.id != null) { + val beatmap = this.osuApi.getBeatmapFromId(mapId) + ?: continue + + userScores.add(mapScore.copy( + beatmap = beatmap.toScoreBeatmap(), + beatmapset = beatmap.beatmapset.toScoreBeatmapSet(), + )) + } + } + + this.logger.info( + "Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}" + ) + + Thread.sleep(SLEEP_AFTER_API_CALL) + } + + } else { + val topUserScores = this.osuApi.getTopUserScores(userId = userId) + val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent") + val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts") + + if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { + this.logger.error("Failed to fetch top scores for user with id = $userId") + this.updateUserQueueService.setUserAsProcessed(userId, failed = true) + continue + } + + userScores += (topUserScores + recentUserScores + firstPlaceUserScores) + + userScores = userScores + .filter { it.beatmap != null && it.beatmapset != null } + .distinctBy { it.best_id } + .toMutableList() + } this.logger.info("Processing user with id = $userId") - this.logger.info("Top scores: ${topUserScores?.size}") - this.logger.info("Recent scores: ${recentUserScores?.size}") - this.logger.info("First place scores: ${firstPlaceUserScores?.size}") + this.logger.info("User has ${userScores.size} total scores") Thread.sleep(SLEEP_AFTER_API_CALL) - if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { - this.logger.error("Failed to fetch top scores for user with id = $userId") - this.updateUserQueueService.setUserAsProcessed(userId, failed = true) - continue - } - - val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores) - .filter { it.beatmap != null && it.beatmapset != null } - .distinctBy { it.best_id } - - this.logger.info("Unique scores: ${allUserScores.size}") + this.logger.info("Unique scores: ${userScores.size}") val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS))) - if(!userExists) { - val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id") - if(apiUser != null) { - this.userService.insertApiUser(apiUser) - this.statistics.usersAddedToDatabase++ - } else { - this.logger.error("Failed to fetch user with id = $userId") - } + if (!userExists) { + this.userService.insertApiUser(user) + this.statistics.usersAddedToDatabase++ } var current = 0 @@ -209,7 +255,7 @@ class ImportScores( .limit(1) .fetchOneInto(OffsetDateTime::class.java) - for(topScore in allUserScores) { + for(topScore in userScores) { val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id)) if (!beatmapExists) { val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id) @@ -264,7 +310,7 @@ class ImportScores( // Update the database dslContext.update(UPDATE_USER_QUEUE) .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current) - .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size) + .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size) .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) .and(UPDATE_USER_QUEUE.PROCESSED.isFalse) .execute() @@ -274,7 +320,7 @@ class ImportScores( lastCompletedUpdate = lastCompletedUpdate, canUpdate = false, progressCurrent = current, - progressTotal = allUserScores.size + progressTotal = userScores.size ) // Update the frontend @@ -289,7 +335,7 @@ class ImportScores( } // Check for stolen replays. - val uniqueBeatmapIds = allUserScores + val uniqueBeatmapIds = userScores .groupBy { it.beatmap!!.id } this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")