Stedos/nise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Processing a user in the queue now downloads all available replays (best effort)

Browse Source
This commit is contained in:
Stedoss 2024-11-18 01:42:50 +00:00
parent 31f301eab2
commit dc846854e4
1 changed files with 78 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt
View File

@ -13,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService
import com.nisemoe.nise.konata.Replay
import com.nisemoe.nise.konata.ReplaySetComparison
import com.nisemoe.nise.konata.compareReplaySet
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.OsuApiModels
import com.nisemoe.nise.osu.*
import com.nisemoe.nise.service.CacheService
import com.nisemoe.nise.service.CompressReplay
import com.nisemoe.nise.service.UpdateUserQueueService
@ -165,38 +163,86 @@ class ImportScores(
}
for(userId in queue) {
val user = this.osuApi.getUserProfile(userId.toString())
if (user == null) {
this.logger.error("Failed to fetch user from queue $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue;
}
var userScores = mutableListOf<OsuApiModels.Score>()
if (user.beatmap_playcounts_count != null) {
val mapsPlayed: MutableSet<Int> = mutableSetOf()
this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
?: break
mapsPlayed.addAll(maps.map { it.beatmap_id })
this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
Thread.sleep(SLEEP_AFTER_API_CALL)
}
var scoreProcessCount = 0
for (mapId in mapsPlayed) {
val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
?: continue
for (mapScore in scores.scores) {
if (mapScore.replay && mapScore.id != null) {
val beatmap = this.osuApi.getBeatmapFromId(mapId)
?: continue
userScores.add(mapScore.copy(
beatmap = beatmap.toScoreBeatmap(),
beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
))
}
}
this.logger.info(
"Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
)
Thread.sleep(SLEEP_AFTER_API_CALL)
}
} else {
val topUserScores = this.osuApi.getTopUserScores(userId = userId)
val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
this.logger.info("Processing user with id = $userId")
this.logger.info("Top scores: ${topUserScores?.size}")
this.logger.info("Recent scores: ${recentUserScores?.size}")
this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
Thread.sleep(SLEEP_AFTER_API_CALL)
if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
this.logger.error("Failed to fetch top scores for user with id = $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue
}
val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
userScores = userScores
.filter { it.beatmap != null && it.beatmapset != null }
.distinctBy { it.best_id }
.toMutableList()
}
this.logger.info("Unique scores: ${allUserScores.size}")
this.logger.info("Processing user with id = $userId")
this.logger.info("User has ${userScores.size} total scores")
Thread.sleep(SLEEP_AFTER_API_CALL)
this.logger.info("Unique scores: ${userScores.size}")
val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
if (!userExists) {
val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
if(apiUser != null) {
this.userService.insertApiUser(apiUser)
this.userService.insertApiUser(user)
this.statistics.usersAddedToDatabase++
} else {
this.logger.error("Failed to fetch user with id = $userId")
}
}
var current = 0
@ -209,7 +255,7 @@ class ImportScores(
.limit(1)
.fetchOneInto(OffsetDateTime::class.java)
for(topScore in allUserScores) {
for(topScore in userScores) {
val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
if (!beatmapExists) {
val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@ -264,7 +310,7 @@ class ImportScores(
// Update the database
dslContext.update(UPDATE_USER_QUEUE)
.set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
.where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
.and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
.execute()
@ -274,7 +320,7 @@ class ImportScores(
lastCompletedUpdate = lastCompletedUpdate,
canUpdate = false,
progressCurrent = current,
progressTotal = allUserScores.size
progressTotal = userScores.size
)
// Update the frontend
@ -289,7 +335,7 @@ class ImportScores(
}
// Check for stolen replays.
val uniqueBeatmapIds = allUserScores
val uniqueBeatmapIds = userScores
.groupBy { it.beatmap!!.id }
this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")