Sync sansei branch #1

Merged
Stedos merged 34 commits from sansei into main 2025-02-24 21:12:28 +00:00
35 changed files with 447 additions and 279 deletions

View File

@ -141,6 +141,7 @@ data class ReplayData(
val beatmap_count_sliders: Int?,
val beatmap_count_spinners: Int?,
val score: Int,
val mods_bitwise: Int,
val mods: List<String>,
val rank: String?,
val ur: Double?,

View File

@ -0,0 +1,22 @@
package com.nisemoe.nise.config
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.boot.jdbc.DataSourceBuilder
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.Primary
import org.springframework.transaction.annotation.EnableTransactionManagement
import javax.sql.DataSource
@Configuration
@EnableTransactionManagement
class DataSourceConfig {
@Primary
@Bean(name = ["niseDataSource"])
@ConfigurationProperties(prefix = "spring.datasource.nise")
fun niseDataSource(): DataSource = DataSourceBuilder.create().build()
@Bean(name = ["replayCacheDataSource"])
@ConfigurationProperties(prefix = "spring.datasource.replay-cache")
fun replayCacheDataSource(): DataSource = DataSourceBuilder.create().build()
}

View File

@ -0,0 +1,22 @@
package com.nisemoe.nise.controller
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RestController
data class HealthResponse(
val healthy: Boolean,
)
val healthResponse = HealthResponse(
healthy = true,
)
@RestController
class HealthController {
@GetMapping("/health")
fun healthCheck(): ResponseEntity<HealthResponse> {
return ResponseEntity.ok(healthResponse)
}
}

View File

@ -0,0 +1,19 @@
package com.nisemoe.nise.controller
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RestController
data class VersionResponse(
val version: String,
)
val versionResponse = VersionResponse(
version = "v20250213",
)
@RestController
class VersionController {
@GetMapping("/version")
fun getVersion(): ResponseEntity<VersionResponse> = ResponseEntity.ok(versionResponse)
}

View File

@ -0,0 +1,46 @@
package com.nisemoe.nise.database
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.stereotype.Service
import javax.sql.DataSource
data class ReplayCacheReplay(
val replayId: Long,
val mapId: Int,
val userId: Int,
val replayData: ByteArray,
val mods: Int,
)
@Service
class ReplayCacheService(
@Qualifier("replayCacheDataSource") private val dataSource: DataSource,
) {
fun getReplayById(replayId: Long): ByteArray? =
dataSource.connection.use { connection ->
val statement = connection.prepareStatement("SELECT replay_data FROM replays WHERE replay_id = ?")
statement.setLong(1, replayId)
val resultSet = statement.executeQuery()
var replayData: ByteArray? = null
while (resultSet.next()) {
replayData = resultSet.getBytes(1)
}
return replayData
}
fun insertReplay(replay: ReplayCacheReplay): Boolean =
dataSource.connection.use { connection ->
val statement = connection.prepareStatement("INSERT INTO replays VALUES (?, ?, ?, ?, ?)")
statement.setLong(1, replay.replayId)
statement.setInt(2, replay.mapId)
statement.setInt(3, replay.userId)
statement.setBytes(4, replay.replayData)
statement.setInt(5, replay.mods)
val updateCount = statement.executeUpdate()
return updateCount != 0
}
}

View File

@ -177,6 +177,8 @@ class ScoreService(
val hitDistribution = this.getHitDistribution(scoreId = result.get(SCORES.ID, Int::class.java))
val charts = this.getCharts(result)
val mods = result.get(SCORES.MODS, Int::class.java)
val replayData = ReplayData(
replay_id = replayId,
user_id = result.get(SCORES.USER_ID, Int::class.java),
@ -204,7 +206,8 @@ class ScoreService(
ur = result.get(SCORES.UR, Double::class.java),
adjusted_ur = result.get(SCORES.ADJUSTED_UR, Double::class.java),
score = result.get(SCORES.SCORE, Int::class.java),
mods = Mod.parseModCombination(result.get(SCORES.MODS, Int::class.java)),
mods_bitwise = mods,
mods = Mod.parseModCombination(mods),
rank = result.get(SCORES.RANK, String::class.java),
snaps = result.get(SCORES.SNAPS, Int::class.java),
hits = result.get(SCORES.EDGE_HITS, Int::class.java),
@ -232,7 +235,7 @@ class ScoreService(
}
fun getDefaultCondition(): Condition {
return SCORES.UR.lessOrEqual(25.0)
return SCORES.UR.lessOrEqual(35.0)
.and(SCORES.IS_BANNED.eq(false))
}

View File

@ -76,6 +76,8 @@ class UserScoreService(
val hitDistribution = this.getHitDistribution(result.get(USER_SCORES.JUDGEMENTS, ByteArray::class.java))
val charts = this.scoreService.getCharts(result)
val mods = result.get(USER_SCORES.MODS, Int::class.java)
val replayData = ReplayData(
replay_id = result.get(USER_SCORES.ONLINE_SCORE_ID, Long::class.java),
username = result.get(USER_SCORES.PLAYER_NAME, String::class.java),
@ -100,7 +102,8 @@ class UserScoreService(
ur = result.get(USER_SCORES.UR, Double::class.java),
adjusted_ur = result.get(USER_SCORES.ADJUSTED_UR, Double::class.java),
score = result.get(USER_SCORES.TOTAL_SCORE, Int::class.java),
mods = Mod.parseModCombination(result.get(USER_SCORES.MODS, Int::class.java)),
mods_bitwise = mods,
mods = Mod.parseModCombination(mods),
snaps = result.get(USER_SCORES.SNAPS, Int::class.java),
hits = result.get(USER_SCORES.EDGE_HITS, Int::class.java),
perfect = result.get(USER_SCORES.PERFECT, Boolean::class.java),

View File

@ -70,13 +70,14 @@ class OsuApi(
.version(HttpClient.Version.HTTP_2)
.build()
fun doRequest(url: String, queryParams: Map<String, Any>, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse<String>? {
fun doRequest(url: String, queryParams: Map<String, Any?>, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse<String>? {
var accessToken: TokenService.AccessTokenResponse? = null
if(authorized)
accessToken = this.tokenService.getAccessToken()
val uriBuilder = StringBuilder(url)
queryParams.forEach { (key, value) ->
if (value != null)
uriBuilder.append("$key=$value&")
}
@ -136,6 +137,19 @@ class OsuApi(
}
}
fun getBeatmapFromId(beatmapId: Int): OsuApiModels.Beatmap? {
val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId", emptyMap())
if (response == null) {
this.logger.info("Error loading beatmap $beatmapId")
return null
}
return when (response.statusCode()) {
200 -> serializer.decodeFromString<OsuApiModels.Beatmap>(response.body())
else -> null
}
}
/**
* Retrieves the replay data for a given score ID from the osu!api.
* Efficiently cycles through the API keys to avoid rate limiting.
@ -213,6 +227,20 @@ class OsuApi(
}
}
fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? {
val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", emptyMap())
if(response == null) {
this.logger.info("Error getting scores on beatmap $beatmapId for user $userId")
return null
}
return when (response.statusCode()) {
200 -> serializer.decodeFromString<OsuApiModels.BeatmapScores>(response.body())
else -> null
}
}
fun searchBeatmapsets(cursor: OsuApiModels.BeatmapsetSearchResultCursor?): OsuApiModels.BeatmapsetSearchResult? {
val queryParams = mutableMapOf(
"s" to "ranked", // Status [only ranked]
@ -234,7 +262,7 @@ class OsuApi(
}
fun checkIfUserBanned(userId: Long): Boolean? {
val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", mapOf())
val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", emptyMap())
if(response == null) {
this.logger.info("Error loading user with userId = $userId")
return null
@ -293,6 +321,24 @@ class OsuApi(
}
}
fun getUserMostPlayed(userId: Long, limit: Int? = null, offset: Int? = null): List<OsuApiModels.BeatmapPlaycount>? {
val queryParams = mapOf(
"limit" to limit,
"offset" to offset,
)
val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/beatmapsets/most_played/?", queryParams)
if (response == null) {
this.logger.info("Error getting user most played ($userId)")
return null
}
return when (response.statusCode()) {
200 -> serializer.decodeFromString<List<OsuApiModels.BeatmapPlaycount>>(response.body())
else -> null
}
}
var rateLimitRemaining: Long = 0L
var rateLimitTotal: Long = 0L

View File

@ -39,6 +39,7 @@ class OsuApiModels {
val avatar_url: String,
val id: Long,
val username: String,
val beatmap_playcounts_count: Int?,
// Documentation: https://osu.ppy.sh/docs/index.html#userextended
val join_date: String?,
@ -204,6 +205,7 @@ class OsuApiModels {
data class Beatmap(
val beatmapset_id: Int,
val difficulty_rating: Double?,
val checksum: String?,
val id: Int,
val version: String?,
val beatmapset: BeatmapSet,
@ -221,6 +223,7 @@ class OsuApiModels {
@Serializable
data class BeatmapSet(
val id: Int,
val artist: String?,
val creator: String?,
val source: String?,
@ -232,4 +235,10 @@ class OsuApiModels {
val content: String
)
@Serializable
data class BeatmapPlaycount(
val beatmap_id: Int,
val count: Int,
)
}

View File

@ -0,0 +1,28 @@
package com.nisemoe.nise.osu
fun OsuApiModels.Beatmap.toScoreBeatmap(): OsuApiModels.ScoreBeatmap =
OsuApiModels.ScoreBeatmap(
id = this.id,
checksum = this.checksum,
difficulty_rating = this.difficulty_rating,
version = this.version,
max_combo = this.max_combo,
total_length = this.total_length,
bpm = this.bpm,
accuracy = this.accuracy,
ar = this.ar,
cs = this.cs,
drain = this.drain,
count_circles = this.count_circles,
count_sliders = this.count_sliders,
count_spinners = this.count_spinners,
)
fun OsuApiModels.BeatmapSet.toScoreBeatmapSet(): OsuApiModels.ScoreBeatmapset =
OsuApiModels.ScoreBeatmapset(
id = this.id,
title = this.title,
artist = this.artist,
creator = this.creator,
source = this.source,
)

View File

@ -3,6 +3,8 @@ package com.nisemoe.nise.scheduler
import com.nisemoe.generated.tables.records.ScoresRecord
import com.nisemoe.generated.tables.references.*
import com.nisemoe.nise.UserQueueDetails
import com.nisemoe.nise.database.ReplayCacheReplay
import com.nisemoe.nise.database.ReplayCacheService
import com.nisemoe.nise.database.ScoreService
import com.nisemoe.nise.database.UserService
import com.nisemoe.nise.integrations.CircleguardService
@ -11,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService
import com.nisemoe.nise.konata.Replay
import com.nisemoe.nise.konata.ReplaySetComparison
import com.nisemoe.nise.konata.compareReplaySet
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.OsuApiModels
import com.nisemoe.nise.osu.*
import com.nisemoe.nise.service.CacheService
import com.nisemoe.nise.service.CompressReplay
import com.nisemoe.nise.service.UpdateUserQueueService
@ -36,6 +36,7 @@ import org.springframework.web.bind.annotation.RestController
import java.time.LocalDateTime
import java.time.OffsetDateTime
import java.time.ZoneOffset
import java.util.Base64
@Service
@RestController
@ -49,8 +50,10 @@ class ImportScores(
private val scoreService: ScoreService,
private val updateUserQueueService: UpdateUserQueueService,
private val circleguardService: CircleguardService,
private val messagingTemplate: SimpMessagingTemplate
private val messagingTemplate: SimpMessagingTemplate,
private val replayCacheService: ReplayCacheService,
) : InitializingBean {
val replayCacheEnabled = (System.getenv("REPLAY_CACHE_ENABLED") ?: "0") != "0"
private val userToUpdateBucket = mutableListOf<Long>()
@ -159,39 +162,93 @@ class ImportScores(
this.logger.info("Processing ${queue.size} users from the queue.")
}
for(userId in queue) {
for(queueEntry in queue) {
val userId = queueEntry.userId
// We should only 'full fetch' a user if they have been explicitly added by another user,
// else we will spend way too much time on random users.
val shouldFullFetch = queueEntry.addedByUserId != null
val user = this.osuApi.getUserProfile(userId.toString())
if (user == null) {
this.logger.error("Failed to fetch user from queue $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue;
}
var userScores = mutableListOf<OsuApiModels.Score>()
if (shouldFullFetch && user.beatmap_playcounts_count != null) {
val mapsPlayed: MutableSet<Int> = mutableSetOf()
this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays")
for (page in 1..(user.beatmap_playcounts_count / 50) + 1) {
val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page)
?: break
mapsPlayed.addAll(maps.map { it.beatmap_id })
this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}")
Thread.sleep(SLEEP_AFTER_API_CALL)
}
var scoreProcessCount = 0
for (mapId in mapsPlayed) {
val scores = this.osuApi.getUserBeatmapScores(userId, mapId)
?: continue
for (mapScore in scores.scores) {
if (mapScore.replay && mapScore.id != null) {
val beatmap = this.osuApi.getBeatmapFromId(mapId)
?: continue
userScores.add(mapScore.copy(
beatmap = beatmap.toScoreBeatmap(),
beatmapset = beatmap.beatmapset.toScoreBeatmapSet(),
))
}
}
this.logger.info(
"Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}"
)
Thread.sleep(SLEEP_AFTER_API_CALL)
}
} else {
val topUserScores = this.osuApi.getTopUserScores(userId = userId)
val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent")
val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts")
this.logger.info("Processing user with id = $userId")
this.logger.info("Top scores: ${topUserScores?.size}")
this.logger.info("Recent scores: ${recentUserScores?.size}")
this.logger.info("First place scores: ${firstPlaceUserScores?.size}")
Thread.sleep(SLEEP_AFTER_API_CALL)
if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) {
this.logger.error("Failed to fetch top scores for user with id = $userId")
this.updateUserQueueService.setUserAsProcessed(userId, failed = true)
continue
}
val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores)
userScores += (topUserScores + recentUserScores + firstPlaceUserScores)
userScores = userScores
.filter { it.beatmap != null && it.beatmapset != null }
.distinctBy { it.best_id }
.toMutableList()
}
this.logger.info("Unique scores: ${allUserScores.size}")
this.logger.info("Processing user with id = $userId")
this.logger.info("User has ${userScores.size} total scores")
Thread.sleep(SLEEP_AFTER_API_CALL)
this.logger.info("Unique scores: ${userScores.size}")
val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS)))
if(!userExists) {
val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id")
if(apiUser != null) {
this.userService.insertApiUser(apiUser)
if (!userExists) {
this.userService.insertApiUser(user)
this.statistics.usersAddedToDatabase++
} else {
this.logger.error("Failed to fetch user with id = $userId")
}
}
var current = 0
@ -204,7 +261,7 @@ class ImportScores(
.limit(1)
.fetchOneInto(OffsetDateTime::class.java)
for(topScore in allUserScores) {
for(topScore in userScores) {
val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id))
if (!beatmapExists) {
val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id)
@ -259,7 +316,7 @@ class ImportScores(
// Update the database
dslContext.update(UPDATE_USER_QUEUE)
.set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size)
.set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size)
.where(UPDATE_USER_QUEUE.USER_ID.eq(userId))
.and(UPDATE_USER_QUEUE.PROCESSED.isFalse)
.execute()
@ -269,7 +326,7 @@ class ImportScores(
lastCompletedUpdate = lastCompletedUpdate,
canUpdate = false,
progressCurrent = current,
progressTotal = allUserScores.size
progressTotal = userScores.size
)
// Update the frontend
@ -284,7 +341,7 @@ class ImportScores(
}
// Check for stolen replays.
val uniqueBeatmapIds = allUserScores
val uniqueBeatmapIds = userScores
.groupBy { it.beatmap!!.id }
this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.")
@ -784,6 +841,24 @@ class ImportScores(
)
}
if (replayCacheEnabled) {
// Insert into replay cache
val replayCacheReplay = ReplayCacheReplay(
score.best_id,
beatmapId,
score.user_id.toInt(),
Base64.getDecoder().decode(scoreReplay.content),
Mod.combineModStrings(score.mods),
)
val replayCacheInsertSuccess = replayCacheService.insertReplay(replayCacheReplay)
if (replayCacheInsertSuccess) {
logger.info("Inserted replay ${score.id} into replay cache")
} else {
logger.error("Could not insert replay ${score.id} into replay cache")
}
}
this.statistics.scoresWithReplayAndAnalyzed++
if (scoreId == null) {

View File

@ -79,12 +79,11 @@ class UpdateUserQueueService(
/**
* Retrieves the full update queue, only pending users.
*/
fun getQueue(): List<Long> {
return dslContext.select(UPDATE_USER_QUEUE.USER_ID)
.from(UPDATE_USER_QUEUE)
fun getQueue(): List<UpdateUserQueueRecord> {
return dslContext.selectFrom(UPDATE_USER_QUEUE)
.where(UPDATE_USER_QUEUE.PROCESSED.isFalse)
.orderBy(UPDATE_USER_QUEUE.CREATED_AT.asc())
.fetchInto(Long::class.java)
.fetch()
}
/**

View File

@ -1,14 +0,0 @@
spring.datasource.url=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public
spring.datasource.username=${POSTGRES_USER:postgres}
spring.datasource.password=${POSTGRES_PASS:postgres}
spring.datasource.driver-class-name=org.postgresql.Driver
spring.datasource.name=HikariPool-PostgreSQL
spring.flyway.enabled=${FLYWAY_ENABLED:true}
spring.flyway.schemas=public
# Batching
spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250
spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048
spring.datasource.hikari.data-source-properties.useServerPrepStmts=true
spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true

View File

@ -33,3 +33,24 @@ spring.security.oauth2.client.provider.osu.authorization-uri=https://osu.ppy.sh/
spring.security.oauth2.client.provider.osu.token-uri=https://osu.ppy.sh/oauth/token
spring.security.oauth2.client.provider.osu.user-info-uri=https://osu.ppy.sh/api/v2/me/osu
spring.security.oauth2.client.provider.osu.user-name-attribute=username
spring.datasource.nise.jdbcUrl=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public
spring.datasource.nise.username=${POSTGRES_USER:postgres}
spring.datasource.nise.password=${POSTGRES_PASS:postgres}
spring.datasource.nise.driver-class-name=org.postgresql.Driver
spring.datasource.nise.name=HikariPool-PostgreSQL
spring.datasource.replay-cache.jdbcUrl=jdbc:postgresql://${REPLAY_CACHE_HOST:postgres}:${REPLAY_CACHE_PORT:5433}/${REPLAY_CACHE_DB:REPLAY_CACHE}?currentSchema=public
spring.datasource.replay-cache.username=${REPLAY_CACHE_USER:postgres}
spring.datasource.replay-cache.password=${REPLAY_CACHE_PASS:postgres}
spring.datasource.replay-cache.driver-class-name=org.postgresql.Driver
spring.datasource.replay-cache.name=HikariPool-PostgreSQL
spring.flyway.enabled=${FLYWAY_ENABLED:true}
spring.flyway.schemas=public
# Batching
spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250
spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048
spring.datasource.hikari.data-source-properties.useServerPrepStmts=true
spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true

View File

@ -1,16 +1,13 @@
FROM python:3.11.8-slim
ENV version=2
ENV PYTHONPATH /app
ENV PYTHONPATH=/app
WORKDIR /app
RUN apt update
COPY requirements.txt ./requirements.txt
RUN pip3 install --upgrade pip && \
pip3 install -r requirements.txt
RUN pip3 install --no-cache-dir -r requirements.txt
# This is *really* bad, but I'd rather get this working rather than forking packages and re-publishing them.
# It'll probably break some day.
@ -22,7 +19,6 @@ RUN sed -i '238s|return \[x for x in arr if lower_limit < x < upper_limit\]|arr_
COPY ./src/ ./src/
ENV GUNICORN_CMD_ARGS="--bind=0.0.0.0:5000 --workers=16"
WORKDIR /app/src
# Run gunicorn with the application
CMD ["gunicorn", "--chdir", "src", "main:app"]
CMD ["python", "main.py"]

View File

@ -1,5 +1,4 @@
ossapi==3.4.3
circleguard==5.4.1
flask==3.0.2
circleguard==5.4.2
brparser==1.0.4
gunicorn==21.2.0
sanic==24.6.0

View File

@ -5,21 +5,21 @@ from dataclasses import dataclass, asdict
from typing import List, Iterable
import numpy as np
from sanic import Request, Sanic, exceptions, json
import scipy
from brparser import Replay, BeatmapOsu, Mod
from circleguard import Circleguard, ReplayString, Hit
from flask import Flask, request, jsonify, abort
from itertools import combinations
from math import isnan
from slider import Beatmap, Circle, Slider, Spinner
from src.WriteStreamWrapper import WriteStreamWrapper
from src.keypresses import get_kp_sliders
from WriteStreamWrapper import WriteStreamWrapper
from keypresses import get_kp_sliders
# Circleguard
cg = Circleguard(os.getenv("OSU_API_KEY"), db_path="./dbs/db.db", slider_dir="./dbs/")
app = Flask(__name__)
app = Sanic(__name__)
def my_filter_outliers(arr, bias=1.5):
"""
@ -123,11 +123,11 @@ class ScoreJudgement:
@app.post("/replay")
def process_replay():
async def process_replay(request: Request):
try:
request_data = request.get_json()
request_data = request.json
if not request_data:
abort(400, description="Bad Request: No JSON data provided.")
raise exceptions.BadRequest("Bad Request: No JSON data provided.")
replay_request = ReplayRequest.from_dict(request_data)
@ -219,10 +219,10 @@ def process_replay():
judgements=judgements
)
return jsonify(ur_response.to_dict())
return json(ur_response.to_dict())
except ValueError as e:
abort(400, description=str(e))
raise exceptions.BadRequest(str(e))
@dataclass
@ -242,11 +242,11 @@ class ReplayDto:
@app.post("/similarity")
def process_similarity():
async def process_similarity(request: Request):
try:
request_data = request.get_json()
request_data = request.json
if not request_data:
abort(400, description="Bad Request: No JSON data provided.")
raise exceptions.BadRequest("Bad Request: No JSON data provided.")
replays: List[ReplayDto] = request_data['replays']
replay_cache = {}
@ -287,11 +287,10 @@ def process_similarity():
)
response.append(new_score_similarity)
return jsonify({'result': response})
return json({'result': response})
except ValueError as e:
abort(400, description=str(e))
raise exceptions.BadRequest(str(e))
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=False)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)

View File

@ -1,4 +1,4 @@
FROM nginx:1.27.0
FROM nginx:1.27.0-alpine
RUN rm -rf /usr/share/nginx/html/*

View File

@ -39,7 +39,7 @@ const routes: Routes = [
{path: 'neko', component: MetabaseComponent, title: 'metabase integration'},
{path: '**', component: HomeComponent, title: '/nise.moe/'},
{path: '**', component: HomeComponent, title: '/nise.stedos.dev/'},
];
@NgModule({

View File

@ -5,13 +5,12 @@
</a>
</div>
<div>
<h2 style="margin-top: 6px">/nise.moe/</h2>
<h2 style="margin-top: 6px">/nise.stedos.dev/</h2>
<ul style="font-size: 15px; line-height: 19px;">
<li><a [routerLink]="['/']">./home</a></li>
<li><a [routerLink]="['/sus']">./suspicious-scores</a></li>
<li><a [routerLink]="['/stolen']">./stolen-replays</a></li>
<li><a [routerLink]="['/search']">./advanced-search</a></li>
<li *ngIf="this.userService.ephemeralUserInfo.showContributions"><a class="link-pink" [routerLink]="['/contribute']">./contribute ♥</a></li>
</ul>
<form (ngSubmit)="onSubmit()">
<input style="width: 100%" type="text" [(ngModel)]="term" [ngModelOptions]="{standalone: true}" id="nise-osu-username" required minlength="2" maxlength="50" placeholder="Search for users...">
@ -35,5 +34,5 @@
<router-outlet></router-outlet>
<div class="text-center version">
v20240511
v20250213
</div>

View File

@ -2,20 +2,15 @@
<div class="subcontainer">
<div class="term">
<h1># Welcome to [nise.moe]</h1>
<h1># Welcome to [nise.stedos.dev] (formerly nise.moe)</h1>
<h3>wtf is this?</h3>
<p>This application will automatically crawl [osu!std] top scores and search for stolen replays or obvious relax/timewarp scores.</p>
<p>It started collecting replays on <i>2024-01-12</i></p>
<p>It is currently in an <i>ALPHA</i> state and will not persist any data. This will change soon.</p>
<p>This website is not affiliated with the osu! game nor ppy. It is an unrelated, unaffiliated, 3rd party project.</p>
<p>If you have any suggestions or want to report bugs, feel free to join the Discord server below.</p>
<div class="text-center mt-4">
<a href="https://discord.gg/wn4gWpA36w" target="_blank" class="btn">Join the Discord!</a>
<a [routerLink]="['/docs']" class="btn api-docs-btn" style="margin-left: 20px;">Check out the /api/ docs.</a>
</div>
<h3 class="mt-4"># do you use rss? (nerd)</h3>
<p>you can keep up with newly detected scores with the rss feed, subscribe to it using your favorite reader.</p>
<div class="text-center">
<a href="https://nise.moe/api/rss.xml" target="_blank">
<a href="https://nise.stedos.dev/api/rss.xml" target="_blank">
<img title="rss-chan!" src="/assets/rss.png" width="64" style="filter: grayscale(40%) sepia(10%) brightness(90%);">
<br>
<span style="padding: 2px; border: 1px dotted #b3b8c3;">

View File

@ -1,12 +1,16 @@
import {UserDetails} from './userDetails';
import {SimilarReplay, SuspiciousScore} from './replays';
import {environment} from "../environments/environment";
export class TextReportService {
static generateTextReportForUserScores(
userDetails: UserDetails,
suspiciousScores: SuspiciousScore[],
similarReplays: SimilarReplay[],
) {
const site = 'nise.stedos.dev';
const detections: string[] = [];
if (suspiciousScores.length > 0) {
@ -21,26 +25,29 @@ export class TextReportService {
report += `Profile: https://osu.ppy.sh/users/${userDetails.user_id}\n`;
for (const suspiciousScore of suspiciousScores) {
report += `\n${this.getRelaxReport(suspiciousScore)}\n`;
report += `\n\n${this.getRelaxReport(suspiciousScore)}\n`;
}
for (const similarReplay of similarReplays) {
report += `\n${this.getStealingReport(similarReplay)}\n`;
report += `\n\n${this.getStealingReport(similarReplay)}\n`;
}
report += `\nGenerated on nise.moe - [${userDetails.username} on nise.moe](https://nise.moe/u/${userDetails.user_id})`;
report += `\n\nGenerated on ${site} - [${userDetails.username} on ${site}](${environment.webUrl}/u/${userDetails.user_id})`;
return report;
}
private static getRelaxReport(suspiciousScore: SuspiciousScore): string {
return `[Replay on ${suspiciousScore.beatmap_title}](https://osu.ppy.sh/scores/osu/${suspiciousScore.replay_id})
cvUR: ${suspiciousScore.ur.toFixed(2)} according to Circleguard`;
}
private static getStealingReport(similarReplay: SimilarReplay): string {
return `[${similarReplay.username_2}'s replay (cheated)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_2})
[${similarReplay.username_1}'s replay (original)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_1})
${similarReplay.similarity.toFixed(2)} similarity according to Circleguard`;
}
}

View File

@ -29,9 +29,9 @@
</a>
</div>
<div class="text-center mt-2">
<!-- <div class="text-center mt-2">
<a class="btn" [href]="'https://replay.nise.moe/' + this.pair.replays[0].replay_id + '/' + this.pair.replays[1].replay_id" target="_blank">Open in Replay Viewer</a>
</div>
</div> -->
<div class="some-page-wrapper text-center">
<div class="row">

View File

@ -53,9 +53,9 @@
Open in CircleGuard
</a>
<a style="flex: 1" class="text-center" [href]="'https://replay.nise.moe/' + this.replayData.replay_id" target="_blank" [class.disabled]="!hasReplay()">
<!-- <a style="flex: 1" class="text-center" [href]="'https://replay.nise.moe/' + this.replayData.replay_id" target="_blank" [class.disabled]="!hasReplay()">
Open in Replay Viewer
</a>
</a> -->
</div>

View File

@ -1,7 +1,7 @@
<div class="main term">
<h1><span class="board">/sus/</span> - Suspicious Scores</h1>
<div class="alert mb-2">
This includes all replays with <25 cvUR. Low values can indicate cheating but always manually review users and
This includes all replays with <35 cvUR. Low values can indicate cheating but always manually review users and
replays before making judgements.
</div>
@ -24,7 +24,7 @@
<input class="form-control" type="number" id="maxPP" [(ngModel)]="this.filterManager.filters.maxPP" (input)="filterScores()"
[readOnly]="this.isUrlFilters" [disabled]="this.isUrlFilters">
</p>
`
<!-- Min cvUR -->
<p>
<label for="minUR" class="form-label">Min cvUR</label>

View File

@ -69,7 +69,7 @@ html {
}
.header {
width: 555px;
width: 600px;
text-align: center;
}

View File

@ -23,8 +23,8 @@ export class ChartHitDistributionComponent implements OnInit, OnChanges {
@Input() mods!: string[];
removeOutliers = true;
groupData = true;
showPercentages = true;
groupData = false;
showPercentages = false;
public barChartLegend = true;
public barChartPlugins = [];

View File

@ -43,8 +43,8 @@ export class ChartComponent implements OnChanges {
@Input() data!: number[];
removeOutliers = true;
groupData = true;
showPercentages = true;
groupData = false;
showPercentages = false;
calculateStatistics(): Array<{ name: string, value: number }> {
if (this.data.length === 0) {

View File

@ -1,5 +1,6 @@
export const environment = {
production: false,
webUrl: 'http://localhost:4200',
apiUrl: 'http://localhost:8080',
wsUrl: 'ws://localhost:8080/websocket',
};

View File

@ -1,5 +1,8 @@
const URL = 'nise.stedos.dev';
export const environment = {
production: true,
apiUrl: 'https://nise.moe/api',
wsUrl: 'wss://nise.moe/api/websocket',
webUrl: `https://${URL}`,
apiUrl: `https://${URL}/api`,
wsUrl: `wss://${URL}/api/websocket`,
};

View File

@ -6,14 +6,14 @@
<title></title>
<link rel="icon" type="image/x-icon" href="/assets/favicon.ico">
<!-- Embed data -->
<meta property="og:title" content="/nise.moe/ - osu!cheaters finder">
<meta property="og:title" content="/nise.stedos.dev/ - osu!cheaters finder">
<meta property="og:description" content="crawls osu!std replays and tries to find naughty boys.">
<meta property="og:url" content="https://nise.moe">
<meta property="og:image" content="https://nise.moe/assets/banner.png">
<meta property="og:url" content="https://nise.stedos.dev">
<meta property="og:image" content="https://nise.stedos.dev/assets/banner.png">
<meta property="og:type" content="website">
<meta name="theme-color" content="#151515">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image:src" content="https://nise.moe/assets/banner.png">
<meta name="twitter:image:src" content="https://nise.stedos.dev/assets/banner.png">
</head>
<body>
<app-root></app-root>

3
nise-infra/Caddyfile Normal file
View File

@ -0,0 +1,3 @@
nise.stedos.dev {
reverse_proxy nise-nginx
}

View File

@ -1,23 +1,25 @@
version: '3'
services:
nginx-main:
image: nginx:latest
container_name: nginx-main
caddy-main:
image: caddy:alpine
container_name: caddy-main
restart: always
volumes:
- ./nginx-main.conf:/etc/nginx/nginx.conf:ro
# nise.moe certificates (by Cloudflare)
- ./nise-data/certificate.pem:/etc/ssl/certs/nisemoe/certificate.pem:ro
- ./nise-data/private.key:/etc/ssl/certs/nisemoe/private.key:ro
- ./Caddyfile:/etc/caddy/Caddyfile:ro
ports:
- "443:443"
- "80:80"
depends_on:
- nise-nginx
# Shared services which are used by others
redis:
image: redis:alpine
container_name: redis
restart: always
postgres:
image: groonga/pgroonga:3.1.6-alpine-15
image: postgres:alpine
container_name: postgres
restart: always
environment:
@ -25,47 +27,6 @@ services:
POSTGRES_PASSWORD: ${DB_PASS}
volumes:
- postgres-data:/var/lib/postgresql/data
command: >
-c shared_buffers=6GB
-c effective_cache_size=12GB
-c work_mem=64MB
-c maintenance_work_mem=2GB
-c checkpoint_completion_target=0.9
-c checkpoint_timeout=15min
-c max_wal_size=2GB
-c wal_buffers=16MB
-c max_connections=100
-c max_worker_processes=8
-c max_parallel_workers_per_gather=4
-c max_parallel_workers=8
-c effective_io_concurrency=40
shm_size: '128mb'
redis:
image: redis:alpine
container_name: redis
restart: always
# ------------------------------------------------------------------
gitea:
image: gitea/gitea
container_name: gitea
restart: always
environment:
USER_UID: 1336
USER_GID: 1336
GITEA__database__DB_TYPE: postgres
GITEA__database__HOST: ${DB_HOST}:5432
GITEA__database__NAME: gitea
GITEA__database__USER: ${DB_USER}
GITEA__database__PASSWD: ${DB_PASS}
depends_on:
- postgres
- redis
volumes:
- ./gitea-data/app.ini:/data/gitea/conf/app.ini
- gitea-data:/data
# ------------------------------------------------------------------
@ -75,19 +36,31 @@ services:
restart: always
volumes:
- ./nise-data/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
- nise-backend
- nise-frontend
nise-circleguard:
image: code.stedos.dev/stedos/nise-circleguard:latest
container_name: nise-circleguard
environment:
OSU_API_KEY: ${OSU_API_KEY}
restart: always
volumes:
- ./nise-data/beatmaps:/app/dbs
nise-backend:
image: git.nise.moe/nuff/nise-backend:latest
image: code.stedos.dev/stedos/nise-backend:latest
container_name: nise-backend
environment:
SPRING_PROFILES_ACTIVE: postgres,discord,import:scores,import:users,fix:scores
SPRING_PROFILES_ACTIVE: postgres,import:scores,import:users,fix:scores
# App configuration
OLD_SCORES_PAGE_SIZE: 1000
# Postgres
POSTGRES_HOST: ${DB_HOST}
POSTGRES_USER: ${DB_USER}
POSTGRES_PASS: ${DB_PASS}
POSTGRES_DB: nise
POSTGRES_DB: ${DB_NAME}
# redis
REDIS_DB: 4
# Discord
@ -97,69 +70,36 @@ services:
OSU_API_KEY: ${OSU_API_KEY}
OSU_CLIENT_ID: ${OSU_CLIENT_ID}
OSU_CLIENT_SECRET: ${OSU_CLIENT_SECRET}
OSU_CALLBACK: "https://nise.moe/api/login/oauth2/code/osu"
OSU_CALLBACK: "https://nise.stedos.dev/api/login/oauth2/code/osu"
# Metabase
METABASE_API_KEY: ${METABASE_API_KEY}
# Internal API
CIRCLEGUARD_API_URL: http://nise-circleguard:5000
# Auth
ORIGIN: "https://nise.moe"
ORIGIN: "https://nise.stedos.dev"
REPLAY_ORIGIN: "https://replay.nise.moe"
COOKIE_SECURE: false
BEATMAPS_PATH: "/app/dbs"
# Replay cache
REPLAY_CACHE_ENABLED: ${REPLAY_CACHE_ENABLED}
REPLAY_CACHE_HOST: ${REPLAY_CACHE_HOST}
REPLAY_CACHE_PORT: ${REPLAY_CACHE_PORT}
REPLAY_CACHE_DB: ${REPLAY_CACHE_DB}
REPLAY_CACHE_USER: ${REPLAY_CACHE_USER}
REPLAY_CACHE_PASS: ${REPLAY_CACHE_PASS}
restart: always
volumes:
- ./nise-data/beatmaps:/app/dbs
depends_on:
- postgres
- redis
- nise-circleguard
nise-circleguard:
image: git.nise.moe/nuff/nise-circleguard:latest
container_name: nise-circleguard
environment:
OSU_API_KEY: ${OSU_API_KEY}
restart: always
volumes:
- ./nise-data/beatmaps:/app/dbs
nise-frontend2:
image: git.nise.moe/nuff/nise-frontend:latest
container_name: nise-frontend2
nise-frontend:
image: code.stedos.dev/stedos/nise-frontend:latest
container_name: nise-frontend
restart: always
nise-replay-viewer:
image: git.nise.moe/nuff/nise-replay-viewer:latest
container_name: nise-replay-viewer
restart: always
nise-discord:
image: git.nise.moe/nuff/nise-discord:latest
container_name: nise-discord
environment:
DISCORD_TOKEN: ${DISCORD_TOKEN}
REACTION_CHANNEL_ID: ${REACTION_CHANNEL_ID}
REACTION_EMOJI_ID: ${REACTION_EMOJI_ID}
restart: always
nise-metabase:
image: metabase/metabase:latest
container_name: nise-metabase
volumes:
- /dev/urandom:/dev/random:ro
environment:
MB_DB_TYPE: postgres
MB_DB_DBNAME: metabase
MB_DB_PORT: 5432
MB_DB_USER: ${DB_METABASE_USER}
MB_DB_PASS: ${DB_METABASE_PASS}
MB_DB_HOST: postgres
healthcheck:
test: curl --fail -I http://localhost:3000/api/health || exit 1
interval: 15s
timeout: 5s
retries: 5
volumes:
postgres-data:
gitea-data:

View File

@ -9,13 +9,13 @@ http {
# Redirect HTTP to HTTPS
server {
listen 80;
server_name nise.moe replay.nise.moe neko.nise.moe;
server_name nise.stedos.dev;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
server_name nise.moe replay.nise.moe git.nise.moe neko.nise.moe;
server_name nise.stedos.dev;
ssl_certificate /etc/ssl/certs/nisemoe/certificate.pem;
ssl_certificate_key /etc/ssl/certs/nisemoe/private.key;

View File

@ -2,29 +2,17 @@ events {}
http {
upstream gitea {
server gitea:3000;
}
upstream nise-frontend {
server nise-frontend2:80;
}
upstream nise-replay-viewer {
server nise-replay-viewer:80;
server nise-frontend:80;
}
upstream nise-backend {
server nise-backend:8080;
}
upstream nise-metabase {
server nise-metabase:3000;
}
server {
listen 80;
server_name nise.moe;
server_name nise.stedos.dev;
location / {
proxy_pass http://nise-frontend;
@ -49,46 +37,4 @@ http {
}
server {
listen 80;
server_name git.nise.moe;
location / {
client_max_body_size 10G;
proxy_pass http://gitea/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
server {
listen 80;
server_name replay.nise.moe;
location / {
proxy_pass http://nise-replay-viewer/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
server {
listen 80;
server_name neko.nise.moe;
location / {
proxy_pass http://nise-metabase/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
}