Merge pull request 'Sync sansei branch' (#1) from sansei into main

Reviewed-on: #1

nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt

@@ -141,6 +141,7 @@ data class ReplayData(
     val beatmap_count_sliders: Int?,
     val beatmap_count_spinners: Int?,
     val score: Int,
+    val mods_bitwise: Int,
     val mods: List<String>,
     val rank: String?,
     val ur: Double?,

nise-backend/src/main/kotlin/com/nisemoe/nise/config/DataSourceConfig.kt

@@ -0,0 +1,22 @@
+package com.nisemoe.nise.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+import org.springframework.boot.jdbc.DataSourceBuilder
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import org.springframework.context.annotation.Primary
+import org.springframework.transaction.annotation.EnableTransactionManagement
+import javax.sql.DataSource
+
+@Configuration
+@EnableTransactionManagement
+class DataSourceConfig {
+    @Primary
+    @Bean(name = ["niseDataSource"])
+    @ConfigurationProperties(prefix = "spring.datasource.nise")
+    fun niseDataSource(): DataSource = DataSourceBuilder.create().build()
+
+    @Bean(name = ["replayCacheDataSource"])
+    @ConfigurationProperties(prefix = "spring.datasource.replay-cache")
+    fun replayCacheDataSource(): DataSource = DataSourceBuilder.create().build()
+}

nise-backend/src/main/kotlin/com/nisemoe/nise/controller/HealthController.kt

@@ -0,0 +1,22 @@
+package com.nisemoe.nise.controller
+
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RestController
+
+data class HealthResponse(
+    val healthy: Boolean,
+)
+
+val healthResponse = HealthResponse(
+    healthy = true,
+)
+
+@RestController
+class HealthController {
+    @GetMapping("/health")
+    fun healthCheck(): ResponseEntity<HealthResponse> {
+        return ResponseEntity.ok(healthResponse)
+    }
+}
+

nise-backend/src/main/kotlin/com/nisemoe/nise/controller/VersionController.kt

@@ -0,0 +1,19 @@
+package com.nisemoe.nise.controller
+
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RestController
+
+data class VersionResponse(
+    val version: String,
+)
+
+val versionResponse = VersionResponse(
+    version = "v20250213",
+)
+
+@RestController
+class VersionController {
+    @GetMapping("/version")
+    fun getVersion(): ResponseEntity<VersionResponse> = ResponseEntity.ok(versionResponse)
+}

nise-backend/src/main/kotlin/com/nisemoe/nise/database/ReplayCacheService.kt

@@ -0,0 +1,46 @@
+package com.nisemoe.nise.database
+
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.stereotype.Service
+import javax.sql.DataSource
+
+data class ReplayCacheReplay(
+    val replayId: Long,
+    val mapId: Int,
+    val userId: Int,
+    val replayData: ByteArray,
+    val mods: Int,
+)
+
+@Service
+class ReplayCacheService(
+    @Qualifier("replayCacheDataSource") private val dataSource: DataSource,
+) {
+    fun getReplayById(replayId: Long): ByteArray? =
+        dataSource.connection.use { connection ->
+            val statement = connection.prepareStatement("SELECT replay_data FROM replays WHERE replay_id = ?")
+            statement.setLong(1, replayId)
+            val resultSet = statement.executeQuery()
+
+            var replayData: ByteArray? = null
+            while (resultSet.next()) {
+                replayData = resultSet.getBytes(1)
+            }
+
+            return replayData
+        }
+
+    fun insertReplay(replay: ReplayCacheReplay): Boolean =
+        dataSource.connection.use { connection ->
+            val statement = connection.prepareStatement("INSERT INTO replays VALUES (?, ?, ?, ?, ?)")
+            statement.setLong(1, replay.replayId)
+            statement.setInt(2, replay.mapId)
+            statement.setInt(3, replay.userId)
+            statement.setBytes(4, replay.replayData)
+            statement.setInt(5, replay.mods)
+
+            val updateCount = statement.executeUpdate()
+
+            return updateCount != 0
+        }
+}

nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt

@@ -177,6 +177,8 @@ class ScoreService(
         val hitDistribution = this.getHitDistribution(scoreId = result.get(SCORES.ID, Int::class.java))
         val charts = this.getCharts(result)
 
+        val mods = result.get(SCORES.MODS, Int::class.java)
+
         val replayData = ReplayData(
             replay_id = replayId,
             user_id = result.get(SCORES.USER_ID, Int::class.java),
@@ -204,7 +206,8 @@ class ScoreService(
             ur = result.get(SCORES.UR, Double::class.java),
             adjusted_ur = result.get(SCORES.ADJUSTED_UR, Double::class.java),
             score = result.get(SCORES.SCORE, Int::class.java),
-            mods = Mod.parseModCombination(result.get(SCORES.MODS, Int::class.java)),
+            mods_bitwise = mods,
+            mods = Mod.parseModCombination(mods),
             rank = result.get(SCORES.RANK, String::class.java),
             snaps = result.get(SCORES.SNAPS, Int::class.java),
             hits = result.get(SCORES.EDGE_HITS, Int::class.java),
@@ -232,7 +235,7 @@ class ScoreService(
     }
 
     fun getDefaultCondition(): Condition {
-        return SCORES.UR.lessOrEqual(25.0)
+        return SCORES.UR.lessOrEqual(35.0)
             .and(SCORES.IS_BANNED.eq(false))
     }
 

nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt

@@ -76,6 +76,8 @@ class UserScoreService(
         val hitDistribution = this.getHitDistribution(result.get(USER_SCORES.JUDGEMENTS, ByteArray::class.java))
         val charts = this.scoreService.getCharts(result)
 
+        val mods = result.get(USER_SCORES.MODS, Int::class.java)
+
         val replayData = ReplayData(
             replay_id = result.get(USER_SCORES.ONLINE_SCORE_ID, Long::class.java),
             username = result.get(USER_SCORES.PLAYER_NAME, String::class.java),
@@ -100,7 +102,8 @@ class UserScoreService(
             ur = result.get(USER_SCORES.UR, Double::class.java),
             adjusted_ur = result.get(USER_SCORES.ADJUSTED_UR, Double::class.java),
             score = result.get(USER_SCORES.TOTAL_SCORE, Int::class.java),
-            mods = Mod.parseModCombination(result.get(USER_SCORES.MODS, Int::class.java)),
+            mods_bitwise = mods,
+            mods = Mod.parseModCombination(mods),
             snaps = result.get(USER_SCORES.SNAPS, Int::class.java),
             hits = result.get(USER_SCORES.EDGE_HITS, Int::class.java),
             perfect = result.get(USER_SCORES.PERFECT, Boolean::class.java),

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt

@@ -70,14 +70,15 @@ class OsuApi(
         .version(HttpClient.Version.HTTP_2)
         .build()
 
-    fun doRequest(url: String, queryParams: Map<String, Any>, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse<String>? {
+    fun doRequest(url: String, queryParams: Map<String, Any?>, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse<String>? {
         var accessToken: TokenService.AccessTokenResponse? = null
         if(authorized)
             accessToken = this.tokenService.getAccessToken()
 
         val uriBuilder = StringBuilder(url)
         queryParams.forEach { (key, value) ->
-            uriBuilder.append("$key=$value&")
+            if (value != null)
+                uriBuilder.append("$key=$value&")
         }
 
         if(appendToUrl != null)
@@ -136,6 +137,19 @@ class OsuApi(
         }
     }
 
+    fun getBeatmapFromId(beatmapId: Int): OsuApiModels.Beatmap? {
+        val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId", emptyMap())
+        if (response == null) {
+            this.logger.info("Error loading beatmap $beatmapId")
+            return null
+        }
+
+        return when (response.statusCode()) {
+            200 -> serializer.decodeFromString<OsuApiModels.Beatmap>(response.body())
+            else -> null
+        }
+    }
+
     /**
      * Retrieves the replay data for a given score ID from the osu!api.
      * Efficiently cycles through the API keys to avoid rate limiting.
@@ -213,6 +227,20 @@ class OsuApi(
         }
     }
 
+    fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? {
+        val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", emptyMap())
+
+        if(response == null) {
+            this.logger.info("Error getting scores on beatmap $beatmapId for user $userId")
+            return null
+        }
+
+        return when (response.statusCode()) {
+            200 -> serializer.decodeFromString<OsuApiModels.BeatmapScores>(response.body())
+            else -> null
+        }
+    }
+
     fun searchBeatmapsets(cursor: OsuApiModels.BeatmapsetSearchResultCursor?): OsuApiModels.BeatmapsetSearchResult? {
         val queryParams = mutableMapOf(
             "s" to "ranked", // Status [only ranked]
@@ -234,7 +262,7 @@ class OsuApi(
     }
 
     fun checkIfUserBanned(userId: Long): Boolean? {
-        val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", mapOf())
+        val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", emptyMap())
         if(response == null) {
             this.logger.info("Error loading user with userId = $userId")
             return null
@@ -293,6 +321,24 @@ class OsuApi(
         }
     }
 
+    fun getUserMostPlayed(userId: Long, limit: Int? = null, offset: Int? = null): List<OsuApiModels.BeatmapPlaycount>? {
+        val queryParams = mapOf(
+            "limit" to limit,
+            "offset" to offset,
+        )
+        val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/beatmapsets/most_played/?", queryParams)
+
+        if (response == null) {
+            this.logger.info("Error getting user most played ($userId)")
+            return null
+        }
+
+        return when (response.statusCode()) {
+            200 -> serializer.decodeFromString<List<OsuApiModels.BeatmapPlaycount>>(response.body())
+            else -> null
+        }
+    }
+
     var rateLimitRemaining: Long = 0L
     var rateLimitTotal: Long = 0L
 

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt

@@ -39,6 +39,7 @@ class OsuApiModels {
         val avatar_url: String,
         val id: Long,
         val username: String,
+        val beatmap_playcounts_count: Int?,
 
         // Documentation: https://osu.ppy.sh/docs/index.html#userextended
         val join_date: String?,
@@ -204,6 +205,7 @@ class OsuApiModels {
     data class Beatmap(
         val beatmapset_id: Int,
         val difficulty_rating: Double?,
+        val checksum: String?,
         val id: Int,
         val version: String?,
         val beatmapset: BeatmapSet,
@@ -221,6 +223,7 @@ class OsuApiModels {
 
     @Serializable
     data class BeatmapSet(
+        val id: Int,
         val artist: String?,
         val creator: String?,
         val source: String?,
@@ -232,4 +235,10 @@ class OsuApiModels {
         val content: String
     )
 
+    @Serializable
+    data class BeatmapPlaycount(
+        val beatmap_id: Int,
+        val count: Int,
+    )
+
 }

nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt

@@ -0,0 +1,28 @@
+package com.nisemoe.nise.osu
+
+fun OsuApiModels.Beatmap.toScoreBeatmap(): OsuApiModels.ScoreBeatmap =
+    OsuApiModels.ScoreBeatmap(
+        id = this.id,
+        checksum = this.checksum,
+        difficulty_rating = this.difficulty_rating,
+        version = this.version,
+        max_combo = this.max_combo,
+        total_length = this.total_length,
+        bpm = this.bpm,
+        accuracy = this.accuracy,
+        ar = this.ar,
+        cs = this.cs,
+        drain = this.drain,
+        count_circles = this.count_circles,
+        count_sliders = this.count_sliders,
+        count_spinners = this.count_spinners,
+    )
+
+fun OsuApiModels.BeatmapSet.toScoreBeatmapSet(): OsuApiModels.ScoreBeatmapset =
+    OsuApiModels.ScoreBeatmapset(
+        id = this.id,
+        title = this.title,
+        artist = this.artist,
+        creator = this.creator,
+        source = this.source,
+    )

nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt

@@ -3,6 +3,8 @@ package com.nisemoe.nise.scheduler
 import com.nisemoe.generated.tables.records.ScoresRecord
 import com.nisemoe.generated.tables.references.*
 import com.nisemoe.nise.UserQueueDetails
+import com.nisemoe.nise.database.ReplayCacheReplay
+import com.nisemoe.nise.database.ReplayCacheService
 import com.nisemoe.nise.database.ScoreService
 import com.nisemoe.nise.database.UserService
 import com.nisemoe.nise.integrations.CircleguardService
@@ -11,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService
 import com.nisemoe.nise.konata.Replay
 import com.nisemoe.nise.konata.ReplaySetComparison
 import com.nisemoe.nise.konata.compareReplaySet
-import com.nisemoe.nise.osu.Mod
-import com.nisemoe.nise.osu.OsuApi
-import com.nisemoe.nise.osu.OsuApiModels
+import com.nisemoe.nise.osu.*
 import com.nisemoe.nise.service.CacheService
 import com.nisemoe.nise.service.CompressReplay
 import com.nisemoe.nise.service.UpdateUserQueueService
@@ -36,6 +36,7 @@ import org.springframework.web.bind.annotation.RestController
 import java.time.LocalDateTime
 import java.time.OffsetDateTime
 import java.time.ZoneOffset
+import java.util.Base64
 
 @Service
 @RestController
@@ -49,8 +50,10 @@ class ImportScores(
     private val scoreService: ScoreService,
     private val updateUserQueueService: UpdateUserQueueService,
     private val circleguardService: CircleguardService,
-    private val messagingTemplate: SimpMessagingTemplate
+    private val messagingTemplate: SimpMessagingTemplate,
+    private val replayCacheService: ReplayCacheService,
 ) : InitializingBean {
+    val replayCacheEnabled = (System.getenv("REPLAY_CACHE_ENABLED") ?: "0") != "0"
 
     private val userToUpdateBucket = mutableListOf<Long>()
 
@@ -159,39 +162,93 @@ class ImportScores(
             this.logger.info("Processing ${queue.size} users from the queue.")
         }
 
-        for(userId in queue) {
-            val topUserScores = this.osuApi.getTopUserScores(userId = userId)
-            val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
-            val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+        for(queueEntry in queue) {
+            val userId = queueEntry.userId
+
+            // We should only 'full fetch' a user if they have been explicitly added by another user,
+            // else we will spend way too much time on random users.
+            val shouldFullFetch = queueEntry.addedByUserId != null
+
+            val user = this.osuApi.getUserProfile(userId.toString())
+
+            if (user == null) {
+                this.logger.error("Failed to fetch user from queue $userId")
+                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                continue;
+            }
+
+            var userScores = mutableListOf<OsuApiModels.Score>()
+
+            if (shouldFullFetch && user.beatmap_playcounts_count != null) {
+                val mapsPlayed: MutableSet<Int> = mutableSetOf()
+
+                this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
+
+                for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
+                    val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
+                        ?: break
+
+                    mapsPlayed.addAll(maps.map { it.beatmap_id })
+
+                    this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+                var scoreProcessCount = 0
+                for (mapId in mapsPlayed) {
+                    val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
+                        ?: continue
+
+                    for (mapScore in scores.scores) {
+                        if (mapScore.replay && mapScore.id != null) {
+                            val beatmap = this.osuApi.getBeatmapFromId(mapId)
+                                ?: continue
+
+                            userScores.add(mapScore.copy(
+                                beatmap = beatmap.toScoreBeatmap(),
+                                beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
+                            ))
+                        }
+                    }
+
+                    this.logger.info(
+                        "Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
+                    )
+
+                    Thread.sleep(SLEEP_AFTER_API_CALL)
+                }
+
+            } else {
+                val topUserScores = this.osuApi.getTopUserScores(userId = userId)
+                val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
+                val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
+
+                if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
+                    this.logger.error("Failed to fetch top scores for user with id = $userId")
+                    this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
+                    continue
+                }
+
+                userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
+
+                userScores = userScores
+                    .filter { it.beatmap != null && it.beatmapset != null }
+                    .distinctBy { it.best_id }
+                    .toMutableList()
+            }
 
             this.logger.info("Processing user with id = $userId")
-            this.logger.info("Top scores: ${topUserScores?.size}")
-            this.logger.info("Recent scores: ${recentUserScores?.size}")
-            this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
+            this.logger.info("User has ${userScores.size} total scores")
 
             Thread.sleep(SLEEP_AFTER_API_CALL)
 
-            if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
-                this.logger.error("Failed to fetch top scores for user with id = $userId")
-                this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
-                continue
-            }
-
-            val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
-                .filter { it.beatmap != null && it.beatmapset != null }
-                .distinctBy { it.best_id }
-
-            this.logger.info("Unique scores: ${allUserScores.size}")
+            this.logger.info("Unique scores: ${userScores.size}")
 
             val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
-            if(!userExists) {
-                val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
-                if(apiUser != null) {
-                    this.userService.insertApiUser(apiUser)
-                    this.statistics.usersAddedToDatabase++
-                } else {
-                    this.logger.error("Failed to fetch user with id = $userId")
-                }
+            if (!userExists) {
+                this.userService.insertApiUser(user)
+                this.statistics.usersAddedToDatabase++
             }
 
             var current = 0
@@ -204,7 +261,7 @@ class ImportScores(
                 .limit(1)
                 .fetchOneInto(OffsetDateTime::class.java)
 
-            for(topScore in allUserScores) {
+            for(topScore in userScores) {
                 val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
                 if (!beatmapExists) {
                     val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@@ -259,7 +316,7 @@ class ImportScores(
                 // Update the database
                 dslContext.update(UPDATE_USER_QUEUE)
                     .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
-                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
+                    .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
                     .where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
                     .and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
                     .execute()
@@ -269,7 +326,7 @@ class ImportScores(
                     lastCompletedUpdate = lastCompletedUpdate,
                     canUpdate = false,
                     progressCurrent = current,
-                    progressTotal = allUserScores.size
+                    progressTotal = userScores.size
                 )
 
                 // Update the frontend
@@ -284,7 +341,7 @@ class ImportScores(
             }
 
             // Check for stolen replays.
-            val uniqueBeatmapIds = allUserScores
+            val uniqueBeatmapIds = userScores
                 .groupBy { it.beatmap!!.id }
 
             this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")
@@ -784,6 +841,24 @@ class ImportScores(
             )
         }
 
+        if (replayCacheEnabled) {
+            // Insert into replay cache
+            val replayCacheReplay = ReplayCacheReplay(
+                score.best_id,
+                beatmapId,
+                score.user_id.toInt(),
+                Base64.getDecoder().decode(scoreReplay.content),
+                Mod.combineModStrings(score.mods),
+            )
+            val replayCacheInsertSuccess = replayCacheService.insertReplay(replayCacheReplay)
+
+            if (replayCacheInsertSuccess) {
+                logger.info("Inserted replay ${score.id} into replay cache")
+            } else {
+                logger.error("Could not insert replay ${score.id} into replay cache")
+            }
+        }
+
         this.statistics.scoresWithReplayAndAnalyzed++
 
         if (scoreId == null) {

nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt

@@ -79,12 +79,11 @@ class UpdateUserQueueService(
     /**
      * Retrieves the full update queue, only pending users.
      */
-    fun getQueue(): List<Long> {
-        return dslContext.select(UPDATE_USER_QUEUE.USER_ID)
-            .from(UPDATE_USER_QUEUE)
+    fun getQueue(): List<UpdateUserQueueRecord> {
+        return dslContext.selectFrom(UPDATE_USER_QUEUE)
             .where(UPDATE_USER_QUEUE.PROCESSED.isFalse)
             .orderBy(UPDATE_USER_QUEUE.CREATED_AT.asc())
-            .fetchInto(Long::class.java)
+            .fetch()
     }
 
     /**

nise-backend/src/main/resources/application-postgres.properties

@@ -1,14 +0,0 @@
-spring.datasource.url=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public
-spring.datasource.username=${POSTGRES_USER:postgres}
-spring.datasource.password=${POSTGRES_PASS:postgres}
-spring.datasource.driver-class-name=org.postgresql.Driver
-spring.datasource.name=HikariPool-PostgreSQL
-
-spring.flyway.enabled=${FLYWAY_ENABLED:true}
-spring.flyway.schemas=public
-
-# Batching
-spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250
-spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048
-spring.datasource.hikari.data-source-properties.useServerPrepStmts=true
-spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true

nise-backend/src/main/resources/application.properties

@@ -32,4 +32,25 @@ spring.security.oauth2.client.registration.osu.provider=osu
 spring.security.oauth2.client.provider.osu.authorization-uri=https://osu.ppy.sh/oauth/authorize
 spring.security.oauth2.client.provider.osu.token-uri=https://osu.ppy.sh/oauth/token
 spring.security.oauth2.client.provider.osu.user-info-uri=https://osu.ppy.sh/api/v2/me/osu
-spring.security.oauth2.client.provider.osu.user-name-attribute=username
+spring.security.oauth2.client.provider.osu.user-name-attribute=username
+
+spring.datasource.nise.jdbcUrl=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public
+spring.datasource.nise.username=${POSTGRES_USER:postgres}
+spring.datasource.nise.password=${POSTGRES_PASS:postgres}
+spring.datasource.nise.driver-class-name=org.postgresql.Driver
+spring.datasource.nise.name=HikariPool-PostgreSQL
+
+spring.datasource.replay-cache.jdbcUrl=jdbc:postgresql://${REPLAY_CACHE_HOST:postgres}:${REPLAY_CACHE_PORT:5433}/${REPLAY_CACHE_DB:REPLAY_CACHE}?currentSchema=public
+spring.datasource.replay-cache.username=${REPLAY_CACHE_USER:postgres}
+spring.datasource.replay-cache.password=${REPLAY_CACHE_PASS:postgres}
+spring.datasource.replay-cache.driver-class-name=org.postgresql.Driver
+spring.datasource.replay-cache.name=HikariPool-PostgreSQL
+
+spring.flyway.enabled=${FLYWAY_ENABLED:true}
+spring.flyway.schemas=public
+
+# Batching
+spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250
+spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048
+spring.datasource.hikari.data-source-properties.useServerPrepStmts=true
+spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true

nise-circleguard/Dockerfile

@@ -1,12 +1,10 @@
 FROM python:3.11.8-slim
 
 ENV version=2
-ENV PYTHONPATH /app
+ENV PYTHONPATH=/app
 
 WORKDIR /app
 
-RUN apt update
-
 COPY requirements.txt ./requirements.txt
 
 RUN pip3 install --no-cache-dir -r requirements.txt
@@ -21,9 +19,6 @@ RUN sed -i '238s|return \[x for x in arr if lower_limit < x < upper_limit\]|arr_
 
 COPY ./src/ ./src/
 
-ENV GUNICORN_CMD_ARGS="--bind=0.0.0.0:5000 --workers=16"
-
 WORKDIR /app/src
 
-# Run gunicorn with the application
-CMD ["sanic", "main", "--port=5000"]
+CMD ["python", "main.py"]

nise-circleguard/requirements.txt

@@ -1,4 +1,4 @@
 ossapi==3.4.3
-circleguard==5.4.1
+circleguard==5.4.2
 brparser==1.0.4
 sanic==24.6.0

nise-circleguard/src/main.py

@@ -291,3 +291,6 @@ async def process_similarity(request: Request):
 
     except ValueError as e:
         raise exceptions.BadRequest(str(e))
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)

nise-frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1.27.0
+FROM nginx:1.27.0-alpine
 
 RUN rm -rf /usr/share/nginx/html/*
 

nise-frontend/src/app/app-routing.module.ts

@@ -39,7 +39,7 @@ const routes: Routes = [
 
   {path: 'neko', component: MetabaseComponent, title: 'metabase integration'},
 
-  {path: '**', component: HomeComponent, title: '/nise.moe/'},
+  {path: '**', component: HomeComponent, title: '/nise.stedos.dev/'},
 ];
 
 @NgModule({

nise-frontend/src/app/app.component.html

@@ -5,13 +5,12 @@
     </a>
   </div>
   <div>
-    <h2 style="margin-top: 6px">/nise.moe/</h2>
+    <h2 style="margin-top: 6px">/nise.stedos.dev/</h2>
     <ul style="font-size: 15px; line-height: 19px;">
       <li><a [routerLink]="['/']">./home</a></li>
       <li><a [routerLink]="['/sus']">./suspicious-scores</a></li>
       <li><a [routerLink]="['/stolen']">./stolen-replays</a></li>
       <li><a [routerLink]="['/search']">./advanced-search</a></li>
-      <li *ngIf="this.userService.ephemeralUserInfo.showContributions"><a class="link-pink" [routerLink]="['/contribute']">./contribute ♥</a></li>
     </ul>
     <form (ngSubmit)="onSubmit()">
       <input style="width: 100%" type="text" [(ngModel)]="term" [ngModelOptions]="{standalone: true}" id="nise-osu-username" required minlength="2" maxlength="50" placeholder="Search for users...">
@@ -35,5 +34,5 @@
 
 <router-outlet></router-outlet>
 <div class="text-center version">
-  v20240511
+  v20250213
 </div>

nise-frontend/src/app/home/home.component.html

@@ -2,20 +2,15 @@
 
   <div class="subcontainer">
     <div class="term">
-      <h1># Welcome to [nise.moe]</h1>
+      <h1># Welcome to [nise.stedos.dev] (formerly nise.moe)</h1>
       <h3>wtf is this?</h3>
       <p>This application will automatically crawl [osu!std] top scores and search for stolen replays or obvious relax/timewarp scores.</p>
-      <p>It started collecting replays on <i>2024-01-12</i></p>
+      <p>It is currently in an <i>ALPHA</i> state and will not persist any data. This will change soon.</p>
       <p>This website is not affiliated with the osu! game nor ppy. It is an unrelated, unaffiliated, 3rd party project.</p>
-      <p>If you have any suggestions or want to report bugs, feel free to join the Discord server below.</p>
-      <div class="text-center mt-4">
-        <a href="https://discord.gg/wn4gWpA36w" target="_blank" class="btn">Join the Discord!</a>
-        <a [routerLink]="['/docs']" class="btn api-docs-btn" style="margin-left: 20px;">Check out the /api/ docs.</a>
-      </div>
       <h3 class="mt-4"># do you use rss? (nerd)</h3>
       <p>you can keep up with newly detected scores with the rss feed, subscribe to it using your favorite reader.</p>
       <div class="text-center">
-        <a href="https://nise.moe/api/rss.xml" target="_blank">
+        <a href="https://nise.stedos.dev/api/rss.xml" target="_blank">
           <img title="rss-chan!" src="/assets/rss.png" width="64" style="filter: grayscale(40%) sepia(10%) brightness(90%);">
           <br>
           <span style="padding: 2px; border: 1px dotted #b3b8c3;">

nise-frontend/src/app/text-report.service.ts

@@ -1,12 +1,16 @@
 import {UserDetails} from './userDetails';
 import {SimilarReplay, SuspiciousScore} from './replays';
+import {environment} from "../environments/environment";
 
 export class TextReportService {
+
   static generateTextReportForUserScores(
     userDetails: UserDetails,
     suspiciousScores: SuspiciousScore[],
     similarReplays: SimilarReplay[],
   ) {
+    const site = 'nise.stedos.dev';
+
     const detections: string[] = [];
 
     if (suspiciousScores.length > 0) {
@@ -21,26 +25,29 @@ export class TextReportService {
     report += `Profile: https://osu.ppy.sh/users/${userDetails.user_id}\n`;
 
     for (const suspiciousScore of suspiciousScores) {
-      report += `\n${this.getRelaxReport(suspiciousScore)}\n`;
+      report += `\n\n${this.getRelaxReport(suspiciousScore)}\n`;
     }
 
     for (const similarReplay of similarReplays) {
-      report += `\n${this.getStealingReport(similarReplay)}\n`;
+      report += `\n\n${this.getStealingReport(similarReplay)}\n`;
     }
 
-    report += `\nGenerated on nise.moe - [${userDetails.username} on nise.moe](https://nise.moe/u/${userDetails.user_id})`;
+    report += `\n\nGenerated on ${site} - [${userDetails.username} on ${site}](${environment.webUrl}/u/${userDetails.user_id})`;
 
     return report;
   }
 
   private static getRelaxReport(suspiciousScore: SuspiciousScore): string {
     return `[Replay on ${suspiciousScore.beatmap_title}](https://osu.ppy.sh/scores/osu/${suspiciousScore.replay_id})
+
 cvUR: ${suspiciousScore.ur.toFixed(2)} according to Circleguard`;
   }
 
   private static getStealingReport(similarReplay: SimilarReplay): string {
     return `[${similarReplay.username_2}'s replay (cheated)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_2})
+
 [${similarReplay.username_1}'s replay (original)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_1})
+
 ${similarReplay.similarity.toFixed(2)} similarity according to Circleguard`;
   }
 }

nise-frontend/src/app/view-replay-pair/view-replay-pair.component.html

@@ -29,9 +29,9 @@
         </a>
       </div>
 
-      <div class="text-center mt-2">
+      <!-- <div class="text-center mt-2">
         <a class="btn" [href]="'https://replay.nise.moe/' + this.pair.replays[0].replay_id + '/' + this.pair.replays[1].replay_id" target="_blank">Open in Replay Viewer</a>
-      </div>
+      </div> -->
 
       <div class="some-page-wrapper text-center">
         <div class="row">

nise-frontend/src/app/view-score/view-score.component.html

@@ -53,9 +53,9 @@
         Open in CircleGuard
       </a>
 
-      <a style="flex: 1" class="text-center" [href]="'https://replay.nise.moe/' + this.replayData.replay_id" target="_blank" [class.disabled]="!hasReplay()">
+      <!-- <a style="flex: 1" class="text-center" [href]="'https://replay.nise.moe/' + this.replayData.replay_id" target="_blank" [class.disabled]="!hasReplay()">
         Open in Replay Viewer
-      </a>
+      </a> -->
 
     </div>
 

nise-frontend/src/app/view-suspicious-scores/view-suspicious-scores.component.html

@@ -1,7 +1,7 @@
 <div class="main term">
   <h1><span class="board">/sus/</span> - Suspicious Scores</h1>
   <div class="alert mb-2">
-    This includes all replays with <25 cvUR. Low values can indicate cheating but always manually review users and
+    This includes all replays with <35 cvUR. Low values can indicate cheating but always manually review users and
     replays before making judgements.
   </div>
 
@@ -24,7 +24,7 @@
       <input class="form-control" type="number" id="maxPP" [(ngModel)]="this.filterManager.filters.maxPP" (input)="filterScores()"
              [readOnly]="this.isUrlFilters" [disabled]="this.isUrlFilters">
     </p>
-
+`
     <!-- Min cvUR -->
     <p>
       <label for="minUR" class="form-label">Min cvUR</label>

nise-frontend/src/assets/style.css

@@ -69,7 +69,7 @@ html {
   }
 
   .header {
-    width: 555px;
+    width: 600px;
     text-align: center;
   }
 

nise-frontend/src/corelib/components/chart-hit-distribution/chart-hit-distribution.component.ts

@@ -23,8 +23,8 @@ export class ChartHitDistributionComponent implements OnInit, OnChanges {
   @Input() mods!: string[];
 
   removeOutliers = true;
-  groupData = true;
-  showPercentages = true;
+  groupData = false;
+  showPercentages = false;
 
   public barChartLegend = true;
   public barChartPlugins = [];

nise-frontend/src/corelib/components/chart/chart.component.ts

@@ -43,8 +43,8 @@ export class ChartComponent implements OnChanges {
   @Input() data!: number[];
 
   removeOutliers = true;
-  groupData = true;
-  showPercentages = true;
+  groupData = false;
+  showPercentages = false;
 
   calculateStatistics(): Array<{ name: string, value: number }> {
     if (this.data.length === 0) {

nise-frontend/src/environments/environment.development.ts

@@ -1,5 +1,6 @@
 export const environment = {
   production: false,
+  webUrl: 'http://localhost:4200',
   apiUrl: 'http://localhost:8080',
   wsUrl: 'ws://localhost:8080/websocket',
 };

nise-frontend/src/environments/environment.ts

@@ -1,5 +1,8 @@
+const URL = 'nise.stedos.dev';
+
 export const environment = {
   production: true,
-  apiUrl: 'https://nise.moe/api',
-  wsUrl: 'wss://nise.moe/api/websocket',
+  webUrl: `https://${URL}`,
+  apiUrl: `https://${URL}/api`,
+  wsUrl: `wss://${URL}/api/websocket`,
 };

nise-frontend/src/index.html

@@ -6,14 +6,14 @@
   <title></title>
   <link rel="icon" type="image/x-icon" href="/assets/favicon.ico">
   <!-- Embed data -->
-  <meta property="og:title" content="/nise.moe/ - osu!cheaters finder">
+  <meta property="og:title" content="/nise.stedos.dev/ - osu!cheaters finder">
   <meta property="og:description" content="crawls osu!std replays and tries to find naughty boys.">
-  <meta property="og:url" content="https://nise.moe">
-  <meta property="og:image" content="https://nise.moe/assets/banner.png">
+  <meta property="og:url" content="https://nise.stedos.dev">
+  <meta property="og:image" content="https://nise.stedos.dev/assets/banner.png">
   <meta property="og:type" content="website">
   <meta name="theme-color" content="#151515">
   <meta name="twitter:card" content="summary_large_image">
-  <meta name="twitter:image:src" content="https://nise.moe/assets/banner.png">
+  <meta name="twitter:image:src" content="https://nise.stedos.dev/assets/banner.png">
 </head>
 <body>
 <app-root></app-root>

nise-infra/Caddyfile

@@ -0,0 +1,3 @@
+nise.stedos.dev {
+    reverse_proxy nise-nginx
+}

nise-infra/docker-compose.yml

@@ -1,23 +1,25 @@
-version: '3'
-
 services:
 
-  nginx-main:
-    image: nginx:latest
-    container_name: nginx-main
+  caddy-main:
+    image: caddy:alpine
+    container_name: caddy-main
     restart: always
     volumes:
-      - ./nginx-main.conf:/etc/nginx/nginx.conf:ro
-      # nise.moe certificates (by Cloudflare)
-      - ./nise-data/certificate.pem:/etc/ssl/certs/nisemoe/certificate.pem:ro
-      - ./nise-data/private.key:/etc/ssl/certs/nisemoe/private.key:ro
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
     ports:
       - "443:443"
       - "80:80"
+    depends_on:
+      - nise-nginx
 
   # Shared services which are used by others
+  redis:
+    image: redis:alpine
+    container_name: redis
+    restart: always
+
   postgres:
-    image: groonga/pgroonga:3.1.6-alpine-15
+    image: postgres:alpine
     container_name: postgres
     restart: always
     environment:
@@ -25,47 +27,6 @@ services:
       POSTGRES_PASSWORD: ${DB_PASS}
     volumes:
       - postgres-data:/var/lib/postgresql/data
-    command: >
-      -c shared_buffers=6GB
-      -c effective_cache_size=12GB
-      -c work_mem=64MB
-      -c maintenance_work_mem=2GB
-      -c checkpoint_completion_target=0.9
-      -c checkpoint_timeout=15min
-      -c max_wal_size=2GB
-      -c wal_buffers=16MB
-      -c max_connections=100
-      -c max_worker_processes=8
-      -c max_parallel_workers_per_gather=4
-      -c max_parallel_workers=8
-      -c effective_io_concurrency=40
-    shm_size: '128mb'
-
-  redis:
-    image: redis:alpine
-    container_name: redis
-    restart: always
-
-  # ------------------------------------------------------------------
-
-  gitea:
-    image: gitea/gitea
-    container_name: gitea
-    restart: always
-    environment:
-      USER_UID: 1336
-      USER_GID: 1336
-      GITEA__database__DB_TYPE: postgres
-      GITEA__database__HOST: ${DB_HOST}:5432
-      GITEA__database__NAME: gitea
-      GITEA__database__USER: ${DB_USER}
-      GITEA__database__PASSWD: ${DB_PASS}
-    depends_on:
-      - postgres
-      - redis
-    volumes:
-      - ./gitea-data/app.ini:/data/gitea/conf/app.ini
-      - gitea-data:/data
 
  # ------------------------------------------------------------------
 
@@ -75,19 +36,31 @@ services:
     restart: always
     volumes:
       - ./nise-data/nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - nise-backend
+      - nise-frontend
+
+  nise-circleguard:
+    image: code.stedos.dev/stedos/nise-circleguard:latest
+    container_name: nise-circleguard
+    environment:
+      OSU_API_KEY: ${OSU_API_KEY}
+    restart: always
+    volumes:
+      - ./nise-data/beatmaps:/app/dbs
 
   nise-backend:
-    image: git.nise.moe/nuff/nise-backend:latest
+    image: code.stedos.dev/stedos/nise-backend:latest
     container_name: nise-backend
     environment:
-      SPRING_PROFILES_ACTIVE: postgres,discord,import:scores,import:users,fix:scores
+      SPRING_PROFILES_ACTIVE: postgres,import:scores,import:users,fix:scores
       # App configuration
       OLD_SCORES_PAGE_SIZE: 1000
       # Postgres
       POSTGRES_HOST: ${DB_HOST}
       POSTGRES_USER: ${DB_USER}
       POSTGRES_PASS: ${DB_PASS}
-      POSTGRES_DB: nise
+      POSTGRES_DB: ${DB_NAME}
       # redis
       REDIS_DB: 4
       # Discord
@@ -97,69 +70,36 @@ services:
       OSU_API_KEY: ${OSU_API_KEY}
       OSU_CLIENT_ID: ${OSU_CLIENT_ID}
       OSU_CLIENT_SECRET: ${OSU_CLIENT_SECRET}
-      OSU_CALLBACK: "https://nise.moe/api/login/oauth2/code/osu"
+      OSU_CALLBACK: "https://nise.stedos.dev/api/login/oauth2/code/osu"
       # Metabase
       METABASE_API_KEY: ${METABASE_API_KEY}
       # Internal API
       CIRCLEGUARD_API_URL: http://nise-circleguard:5000
       # Auth
-      ORIGIN: "https://nise.moe"
+      ORIGIN: "https://nise.stedos.dev"
       REPLAY_ORIGIN: "https://replay.nise.moe"
       COOKIE_SECURE: false
       BEATMAPS_PATH: "/app/dbs"
+      # Replay cache
+      REPLAY_CACHE_ENABLED: ${REPLAY_CACHE_ENABLED}
+      REPLAY_CACHE_HOST: ${REPLAY_CACHE_HOST}
+      REPLAY_CACHE_PORT: ${REPLAY_CACHE_PORT}
+      REPLAY_CACHE_DB: ${REPLAY_CACHE_DB}
+      REPLAY_CACHE_USER: ${REPLAY_CACHE_USER}
+      REPLAY_CACHE_PASS: ${REPLAY_CACHE_PASS}
     restart: always
     volumes:
         - ./nise-data/beatmaps:/app/dbs
     depends_on:
       - postgres
       - redis
+      - nise-circleguard
 
-  nise-circleguard:
-    image: git.nise.moe/nuff/nise-circleguard:latest
-    container_name: nise-circleguard
-    environment:
-      OSU_API_KEY: ${OSU_API_KEY}
-    restart: always
-    volumes:
-      - ./nise-data/beatmaps:/app/dbs
-
-  nise-frontend2:
-    image: git.nise.moe/nuff/nise-frontend:latest
-    container_name: nise-frontend2
+  nise-frontend:
+    image: code.stedos.dev/stedos/nise-frontend:latest
+    container_name: nise-frontend
     restart: always
 
-  nise-replay-viewer:
-    image: git.nise.moe/nuff/nise-replay-viewer:latest
-    container_name: nise-replay-viewer
-    restart: always
-
-  nise-discord:
-    image: git.nise.moe/nuff/nise-discord:latest
-    container_name: nise-discord
-    environment:
-      DISCORD_TOKEN: ${DISCORD_TOKEN}
-      REACTION_CHANNEL_ID: ${REACTION_CHANNEL_ID}
-      REACTION_EMOJI_ID: ${REACTION_EMOJI_ID}
-    restart: always
-
-  nise-metabase:
-    image: metabase/metabase:latest
-    container_name: nise-metabase
-    volumes:
-    - /dev/urandom:/dev/random:ro
-    environment:
-      MB_DB_TYPE: postgres
-      MB_DB_DBNAME: metabase
-      MB_DB_PORT: 5432
-      MB_DB_USER: ${DB_METABASE_USER}
-      MB_DB_PASS: ${DB_METABASE_PASS}
-      MB_DB_HOST: postgres
-    healthcheck:
-      test: curl --fail -I http://localhost:3000/api/health || exit 1
-      interval: 15s
-      timeout: 5s
-      retries: 5
 
 volumes:
-  postgres-data:
-  gitea-data:
+  postgres-data:

nise-infra/nginx-main.conf

@@ -9,13 +9,13 @@ http {
     # Redirect HTTP to HTTPS
     server {
         listen 80;
-        server_name nise.moe replay.nise.moe neko.nise.moe;
+        server_name nise.stedos.dev;
         return 301 https://$host$request_uri;
     }
 
     server {
         listen 443 ssl;
-        server_name nise.moe replay.nise.moe git.nise.moe neko.nise.moe;
+        server_name nise.stedos.dev;
 
         ssl_certificate /etc/ssl/certs/nisemoe/certificate.pem;
         ssl_certificate_key /etc/ssl/certs/nisemoe/private.key;

nise-infra/nise-data/nginx.conf

@@ -2,29 +2,17 @@ events {}
 
 http {
 
-    upstream gitea {
-        server gitea:3000;
-    }
-
     upstream nise-frontend {
-        server nise-frontend2:80;
-    }
-
-    upstream nise-replay-viewer {
-        server nise-replay-viewer:80;
+        server nise-frontend:80;
     }
 
     upstream nise-backend {
         server nise-backend:8080;
     }
 
-    upstream nise-metabase {
-        server nise-metabase:3000;
-    }
-
     server {
         listen 80;
-        server_name nise.moe;
+        server_name nise.stedos.dev;
 
         location / {
             proxy_pass http://nise-frontend;
@@ -49,46 +37,4 @@ http {
 
     }
 
-    server {
-        listen 80;
-
-        server_name git.nise.moe;
-
-        location / { 
-            client_max_body_size 10G;
-            proxy_pass http://gitea/;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-        }
-
-    }
-
-    server {
-        listen 80;
-        server_name replay.nise.moe;
-
-        location / { 
-            proxy_pass http://nise-replay-viewer/;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-        }
-    }
-
-    server {
-        listen 80;
-        server_name neko.nise.moe;
-
-        location / { 
-            proxy_pass http://nise-metabase/;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $scheme;
-        }
-    }
-
 }