Stedos/nise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Stedos:dc846854e4b350f979e63ab79d2a4bb892cf16e0
Branches Tags
Stedos:main
Stedos:sansei
Stedos:feature/caddy
Stedos:sanic-cg-service
..
compare: Stedos:e0cabfefcf3045751c58078bab2c79700838e7f5
Branches Tags
Stedos:sansei
Stedos:main
Stedos:feature/caddy
Stedos:sanic-cg-service

No commits in common. "dc846854e4b350f979e63ab79d2a4bb892cf16e0" and "e0cabfefcf3045751c58078bab2c79700838e7f5" have entirely different histories.
dc846854e4 ... e0cabfefcf

4 changed files with 34 additions and 123 deletions
Show Stats Expand all files Collapse all files

17
nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt
View File

@ -137,19 +137,6 @@ class OsuApi(
}
}
fun getBeatmapFromId(beatmapId: Int): OsuApiModels.Beatmap? {
val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId", emptyMap())
if (response == null) {
this.logger.info("Error loading beatmap $beatmapId")
return null
}
return when (response.statusCode()) {
200 -> serializer.decodeFromString<OsuApiModels.Beatmap>(response.body())
else -> null
}
}
/**
* Retrieves the replay data for a given score ID from the osu!api.
* Efficiently cycles through the API keys to avoid rate limiting.
@ -228,7 +215,7 @@ class OsuApi(
}
fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? {
val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", emptyMap())
val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", mapOf())
if(response == null) {
this.logger.info("Error getting scores on beatmap $beatmapId for user $userId")
@ -262,7 +249,7 @@ class OsuApi(
}
fun checkIfUserBanned(userId: Long): Boolean? {
val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", emptyMap())
val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", mapOf())
if(response == null) {
this.logger.info("Error loading user with userId = $userId")
return null

2
nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt
View File

@ -205,7 +205,6 @@ class OsuApiModels {
data class Beatmap(
val beatmapset_id: Int,
val difficulty_rating: Double?,
val checksum: String?,
val id: Int,
val version: String?,
val beatmapset: BeatmapSet,
@ -223,7 +222,6 @@ class OsuApiModels {
@Serializable
data class BeatmapSet(
val id: Int,
val artist: String?,
val creator: String?,
val source: String?,

28
nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt
View File

@ -1,28 +0,0 @@
package com.nisemoe.nise.osu
fun OsuApiModels.Beatmap.toScoreBeatmap(): OsuApiModels.ScoreBeatmap =
OsuApiModels.ScoreBeatmap(
id = this.id,
checksum = this.checksum,
difficulty_rating = this.difficulty_rating,
version = this.version,
max_combo = this.max_combo,
total_length = this.total_length,
bpm = this.bpm,
accuracy = this.accuracy,
ar = this.ar,
cs = this.cs,
drain = this.drain,
count_circles = this.count_circles,
count_sliders = this.count_sliders,
count_spinners = this.count_spinners,
)
fun OsuApiModels.BeatmapSet.toScoreBeatmapSet(): OsuApiModels.ScoreBeatmapset =
OsuApiModels.ScoreBeatmapset(
id = this.id,
title = this.title,
artist = this.artist,
creator = this.creator,
source = this.source,
)

90
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt
View File

@ -13,7 +13,9 @@ import com.nisemoe.nise.integrations.DiscordService
import com.nisemoe.nise.konata.Replay
import com.nisemoe.nise.konata.ReplaySetComparison
import com.nisemoe.nise.konata.compareReplaySet
import com.nisemoe.nise.osu.*
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.OsuApiModels
import com.nisemoe.nise.service.CacheService
import com.nisemoe.nise.service.CompressReplay
import com.nisemoe.nise.service.UpdateUserQueueService
@ -163,86 +165,38 @@ class ImportScores(
}
for(userId in queue) {
val user = this.osuApi.getUserProfile(userId.toString())
if (user == null) {
this.logger.error("Failed to fetch user from queue $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue;
}
var userScores = mutableListOf<OsuApiModels.Score>()
if (user.beatmap_playcounts_count != null) {
val mapsPlayed: MutableSet<Int> = mutableSetOf()
this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
?: break
mapsPlayed.addAll(maps.map { it.beatmap_id })
this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
Thread.sleep(SLEEP_AFTER_API_CALL)
}
var scoreProcessCount = 0
for (mapId in mapsPlayed) {
val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
?: continue
for (mapScore in scores.scores) {
if (mapScore.replay && mapScore.id != null) {
val beatmap = this.osuApi.getBeatmapFromId(mapId)
?: continue
userScores.add(mapScore.copy(
beatmap = beatmap.toScoreBeatmap(),
beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
))
}
}
this.logger.info(
"Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
)
Thread.sleep(SLEEP_AFTER_API_CALL)
}
} else {
val topUserScores = this.osuApi.getTopUserScores(userId = userId)
val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
this.logger.info("Processing user with id = $userId")
this.logger.info("Top scores: ${topUserScores?.size}")
this.logger.info("Recent scores: ${recentUserScores?.size}")
this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
Thread.sleep(SLEEP_AFTER_API_CALL)
if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
this.logger.error("Failed to fetch top scores for user with id = $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue
}
userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
userScores = userScores
val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
.filter { it.beatmap != null && it.beatmapset != null }
.distinctBy { it.best_id }
.toMutableList()
}
this.logger.info("Processing user with id = $userId")
this.logger.info("User has ${userScores.size} total scores")
Thread.sleep(SLEEP_AFTER_API_CALL)
this.logger.info("Unique scores: ${userScores.size}")
this.logger.info("Unique scores: ${allUserScores.size}")
val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
if(!userExists) {
this.userService.insertApiUser(user)
val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
if(apiUser != null) {
this.userService.insertApiUser(apiUser)
this.statistics.usersAddedToDatabase++
} else {
this.logger.error("Failed to fetch user with id = $userId")
}
}
var current = 0
@ -255,7 +209,7 @@ class ImportScores(
.limit(1)
.fetchOneInto(OffsetDateTime::class.java)
for(topScore in userScores) {
for(topScore in allUserScores) {
val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
if (!beatmapExists) {
val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@ -310,7 +264,7 @@ class ImportScores(
// Update the database
dslContext.update(UPDATE_USER_QUEUE)
.set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
.where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
.and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
.execute()
@ -320,7 +274,7 @@ class ImportScores(
lastCompletedUpdate = lastCompletedUpdate,
canUpdate = false,
progressCurrent = current,
progressTotal = userScores.size
progressTotal = allUserScores.size
)
// Update the frontend
@ -335,7 +289,7 @@ class ImportScores(
}
// Check for stolen replays.
val uniqueBeatmapIds = userScores
val uniqueBeatmapIds = allUserScores
.groupBy { it.beatmap!!.id }
this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")