From 81fd373f5c160b02b5655712626869bf090087cd Mon Sep 17 00:00:00 2001 From: Stedoss Date: Fri, 1 Nov 2024 04:00:49 +0000 Subject: [PATCH 01/34] nise-circleguard: Update to use sanic instead of Flask --- nise-circleguard/Dockerfile | 7 ++++--- nise-circleguard/requirements.txt | 3 +-- nise-circleguard/src/main.py | 32 ++++++++++++++----------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/nise-circleguard/Dockerfile b/nise-circleguard/Dockerfile index 9eed673..0d11025 100644 --- a/nise-circleguard/Dockerfile +++ b/nise-circleguard/Dockerfile @@ -9,8 +9,7 @@ RUN apt update COPY requirements.txt ./requirements.txt -RUN pip3 install --upgrade pip && \ - pip3 install -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements.txt # This is *really* bad, but I'd rather get this working rather than forking packages and re-publishing them. # It'll probably break some day. @@ -24,5 +23,7 @@ COPY ./src/ ./src/ ENV GUNICORN_CMD_ARGS="--bind=0.0.0.0:5000 --workers=16" +WORKDIR /app/src + # Run gunicorn with the application -CMD ["gunicorn", "--chdir", "src", "main:app"] \ No newline at end of file +CMD ["sanic", "main", "--port=5000"] diff --git a/nise-circleguard/requirements.txt b/nise-circleguard/requirements.txt index 45a9b69..a070b01 100644 --- a/nise-circleguard/requirements.txt +++ b/nise-circleguard/requirements.txt @@ -1,5 +1,4 @@ ossapi==3.4.3 circleguard==5.4.1 -flask==3.0.2 brparser==1.0.4 -gunicorn==21.2.0 \ No newline at end of file +sanic==24.6.0 diff --git a/nise-circleguard/src/main.py b/nise-circleguard/src/main.py index 6c1adae..faeaf15 100644 --- a/nise-circleguard/src/main.py +++ b/nise-circleguard/src/main.py @@ -5,21 +5,21 @@ from dataclasses import dataclass, asdict from typing import List, Iterable import numpy as np +from sanic import Request, Sanic, exceptions, json import scipy from brparser import Replay, BeatmapOsu, Mod from circleguard import Circleguard, ReplayString, Hit -from flask import Flask, request, jsonify, abort from itertools import combinations from math import isnan from slider import Beatmap, Circle, Slider, Spinner -from src.WriteStreamWrapper import WriteStreamWrapper -from src.keypresses import get_kp_sliders +from WriteStreamWrapper import WriteStreamWrapper +from keypresses import get_kp_sliders # Circleguard cg = Circleguard(os.getenv("OSU_API_KEY"), db_path="./dbs/db.db", slider_dir="./dbs/") -app = Flask(__name__) +app = Sanic(__name__) def my_filter_outliers(arr, bias=1.5): """ @@ -123,11 +123,11 @@ class ScoreJudgement: @app.post("/replay") -def process_replay(): +async def process_replay(request: Request): try: - request_data = request.get_json() + request_data = request.json if not request_data: - abort(400, description="Bad Request: No JSON data provided.") + raise exceptions.BadRequest("Bad Request: No JSON data provided.") replay_request = ReplayRequest.from_dict(request_data) @@ -219,10 +219,10 @@ def process_replay(): judgements=judgements ) - return jsonify(ur_response.to_dict()) + return json(ur_response.to_dict()) except ValueError as e: - abort(400, description=str(e)) + raise exceptions.BadRequest(str(e)) @dataclass @@ -242,11 +242,11 @@ class ReplayDto: @app.post("/similarity") -def process_similarity(): +async def process_similarity(request: Request): try: - request_data = request.get_json() + request_data = request.json if not request_data: - abort(400, description="Bad Request: No JSON data provided.") + raise exceptions.BadRequest("Bad Request: No JSON data provided.") replays: List[ReplayDto] = request_data['replays'] replay_cache = {} @@ -287,11 +287,7 @@ def process_similarity(): ) response.append(new_score_similarity) - return jsonify({'result': response}) + return json({'result': response}) except ValueError as e: - abort(400, description=str(e)) - - -if __name__ == "__main__": - app.run(host='0.0.0.0', debug=False) + raise exceptions.BadRequest(str(e)) From e28d3e72116bc03ec4efb04edb75e48d54c31b12 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sat, 2 Nov 2024 20:37:28 +0000 Subject: [PATCH 02/34] Add support for multiple data source beans Also implements the connection needed for the replay cache --- .../nisemoe/nise/config/DataSourceConfig.kt | 22 ++++++++++++++++++ .../resources/application-postgres.properties | 14 ----------- .../src/main/resources/application.properties | 23 ++++++++++++++++++- 3 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/config/DataSourceConfig.kt delete mode 100644 nise-backend/src/main/resources/application-postgres.properties diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/config/DataSourceConfig.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/config/DataSourceConfig.kt new file mode 100644 index 0000000..4612594 --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/config/DataSourceConfig.kt @@ -0,0 +1,22 @@ +package com.nisemoe.nise.config + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.jdbc.DataSourceBuilder +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Primary +import org.springframework.transaction.annotation.EnableTransactionManagement +import javax.sql.DataSource + +@Configuration +@EnableTransactionManagement +class DataSourceConfig { + @Primary + @Bean(name = ["niseDataSource"]) + @ConfigurationProperties(prefix = "spring.datasource.nise") + fun niseDataSource(): DataSource = DataSourceBuilder.create().build() + + @Bean(name = ["replayCacheDataSource"]) + @ConfigurationProperties(prefix = "spring.datasource.replay-cache") + fun replayCacheDataSource(): DataSource = DataSourceBuilder.create().build() +} diff --git a/nise-backend/src/main/resources/application-postgres.properties b/nise-backend/src/main/resources/application-postgres.properties deleted file mode 100644 index cfe8ac2..0000000 --- a/nise-backend/src/main/resources/application-postgres.properties +++ /dev/null @@ -1,14 +0,0 @@ -spring.datasource.url=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public -spring.datasource.username=${POSTGRES_USER:postgres} -spring.datasource.password=${POSTGRES_PASS:postgres} -spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.name=HikariPool-PostgreSQL - -spring.flyway.enabled=${FLYWAY_ENABLED:true} -spring.flyway.schemas=public - -# Batching -spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250 -spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048 -spring.datasource.hikari.data-source-properties.useServerPrepStmts=true -spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true diff --git a/nise-backend/src/main/resources/application.properties b/nise-backend/src/main/resources/application.properties index 94714d4..45432b5 100644 --- a/nise-backend/src/main/resources/application.properties +++ b/nise-backend/src/main/resources/application.properties @@ -32,4 +32,25 @@ spring.security.oauth2.client.registration.osu.provider=osu spring.security.oauth2.client.provider.osu.authorization-uri=https://osu.ppy.sh/oauth/authorize spring.security.oauth2.client.provider.osu.token-uri=https://osu.ppy.sh/oauth/token spring.security.oauth2.client.provider.osu.user-info-uri=https://osu.ppy.sh/api/v2/me/osu -spring.security.oauth2.client.provider.osu.user-name-attribute=username \ No newline at end of file +spring.security.oauth2.client.provider.osu.user-name-attribute=username + +spring.datasource.nise.jdbcUrl=jdbc:postgresql://${POSTGRES_HOST:postgres}:${POSTGRES_PORT:5432}/${POSTGRES_DB:postgres}?currentSchema=public +spring.datasource.nise.username=${POSTGRES_USER:postgres} +spring.datasource.nise.password=${POSTGRES_PASS:postgres} +spring.datasource.nise.driver-class-name=org.postgresql.Driver +spring.datasource.nise.name=HikariPool-PostgreSQL + +spring.datasource.replay-cache.jdbcUrl=jdbc:postgresql://${REPLAY_CACHE_HOST:postgres}:${REPLAY_CACHE_PORT:5433}/${REPLAY_CACHE_DB:REPLAY_CACHE}?currentSchema=public +spring.datasource.replay-cache.username=${REPLAY_CACHE_USER:postgres} +spring.datasource.replay-cache.password=${REPLAY_CACHE_PASS:postgres} +spring.datasource.replay-cache.driver-class-name=org.postgresql.Driver +spring.datasource.replay-cache.name=HikariPool-PostgreSQL + +spring.flyway.enabled=${FLYWAY_ENABLED:true} +spring.flyway.schemas=public + +# Batching +spring.datasource.hikari.data-source-properties.prepStmtCacheSize=250 +spring.datasource.hikari.data-source-properties.prepStmtCacheSqlLimit=2048 +spring.datasource.hikari.data-source-properties.useServerPrepStmts=true +spring.datasource.hikari.data-source-properties.rewriteBatchedStatements=true From 073966745e8dba06961dca6583a2c35306d6e695 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sat, 2 Nov 2024 20:43:20 +0000 Subject: [PATCH 03/34] Add `ReplayCacheService` to allow for communication with the replay cache --- .../nise/database/ReplayCacheService.kt | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/database/ReplayCacheService.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ReplayCacheService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ReplayCacheService.kt new file mode 100644 index 0000000..f78d68d --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ReplayCacheService.kt @@ -0,0 +1,46 @@ +package com.nisemoe.nise.database + +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.stereotype.Service +import javax.sql.DataSource + +data class ReplayCacheReplay( + val replayId: Long, + val mapId: Int, + val userId: Int, + val replayData: ByteArray, + val mods: Int, +) + +@Service +class ReplayCacheService( + @Qualifier("replayCacheDataSource") private val dataSource: DataSource, +) { + fun getReplayById(replayId: Long): ByteArray? = + dataSource.connection.use { connection -> + val statement = connection.prepareStatement("SELECT replay_data FROM replays WHERE replay_id = ?") + statement.setLong(1, replayId) + val resultSet = statement.executeQuery() + + var replayData: ByteArray? = null + while (resultSet.next()) { + replayData = resultSet.getBytes(1) + } + + return replayData + } + + fun insertReplay(replay: ReplayCacheReplay): Boolean = + dataSource.connection.use { connection -> + val statement = connection.prepareStatement("INSERT INTO replays VALUES (?, ?, ?, ?, ?)") + statement.setLong(1, replay.replayId) + statement.setInt(2, replay.mapId) + statement.setInt(3, replay.userId) + statement.setBytes(4, replay.replayData) + statement.setInt(5, replay.mods) + + val updateCount = statement.executeUpdate() + + return updateCount != 0 + } +} From 5d44d76671f833ff65448fcb6aa31a713cb63308 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sat, 2 Nov 2024 20:43:35 +0000 Subject: [PATCH 04/34] Attempt to add replay to replay cache on importing scores --- .../nisemoe/nise/scheduler/ImportScores.kt | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index df2af9d..7052e9a 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -3,6 +3,8 @@ package com.nisemoe.nise.scheduler import com.nisemoe.generated.tables.records.ScoresRecord import com.nisemoe.generated.tables.references.* import com.nisemoe.nise.UserQueueDetails +import com.nisemoe.nise.database.ReplayCacheReplay +import com.nisemoe.nise.database.ReplayCacheService import com.nisemoe.nise.database.ScoreService import com.nisemoe.nise.database.UserService import com.nisemoe.nise.integrations.CircleguardService @@ -36,6 +38,7 @@ import org.springframework.web.bind.annotation.RestController import java.time.LocalDateTime import java.time.OffsetDateTime import java.time.ZoneOffset +import java.util.Base64 @Service @RestController @@ -49,7 +52,8 @@ class ImportScores( private val scoreService: ScoreService, private val updateUserQueueService: UpdateUserQueueService, private val circleguardService: CircleguardService, - private val messagingTemplate: SimpMessagingTemplate + private val messagingTemplate: SimpMessagingTemplate, + private val replayCacheService: ReplayCacheService, ) : InitializingBean { private val userToUpdateBucket = mutableListOf() @@ -784,6 +788,22 @@ class ImportScores( ) } + // Insert into replay cache + val replayCacheReplay = ReplayCacheReplay( + score.best_id, + beatmapId, + score.user_id.toInt(), + Base64.getDecoder().decode(scoreReplay.content), + Mod.combineModStrings(score.mods), + ) + val replayCacheInsertSuccess = replayCacheService.insertReplay(replayCacheReplay) + + if (replayCacheInsertSuccess) { + logger.info("Inserted replay ${score.id} into replay cache") + } else { + logger.error("Could not insert replay ${score.id} into replay cache") + } + this.statistics.scoresWithReplayAndAnalyzed++ if (scoreId == null) { From da6aefa74d51fc547a74f1d5882719732372ef44 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sun, 3 Nov 2024 15:22:33 +0000 Subject: [PATCH 05/34] Allow replay cache to be disabled with env variable --- .../nisemoe/nise/scheduler/ImportScores.kt | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index 7052e9a..1c60490 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -55,6 +55,7 @@ class ImportScores( private val messagingTemplate: SimpMessagingTemplate, private val replayCacheService: ReplayCacheService, ) : InitializingBean { + val replayCacheEnabled = (System.getenv("REPLAY_CACHE_ENABLED") ?: "0") != "0" private val userToUpdateBucket = mutableListOf() @@ -788,20 +789,22 @@ class ImportScores( ) } - // Insert into replay cache - val replayCacheReplay = ReplayCacheReplay( - score.best_id, - beatmapId, - score.user_id.toInt(), - Base64.getDecoder().decode(scoreReplay.content), - Mod.combineModStrings(score.mods), - ) - val replayCacheInsertSuccess = replayCacheService.insertReplay(replayCacheReplay) + if (replayCacheEnabled) { + // Insert into replay cache + val replayCacheReplay = ReplayCacheReplay( + score.best_id, + beatmapId, + score.user_id.toInt(), + Base64.getDecoder().decode(scoreReplay.content), + Mod.combineModStrings(score.mods), + ) + val replayCacheInsertSuccess = replayCacheService.insertReplay(replayCacheReplay) - if (replayCacheInsertSuccess) { - logger.info("Inserted replay ${score.id} into replay cache") - } else { - logger.error("Could not insert replay ${score.id} into replay cache") + if (replayCacheInsertSuccess) { + logger.info("Inserted replay ${score.id} into replay cache") + } else { + logger.error("Could not insert replay ${score.id} into replay cache") + } } this.statistics.scoresWithReplayAndAnalyzed++ From 67150e7e42dda57616103d79f67fad3d9f0c80cc Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sun, 3 Nov 2024 15:52:33 +0000 Subject: [PATCH 06/34] Avoid warning in `nise-circleguard` dockerfile --- nise-circleguard/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-circleguard/Dockerfile b/nise-circleguard/Dockerfile index 0d11025..93ba737 100644 --- a/nise-circleguard/Dockerfile +++ b/nise-circleguard/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11.8-slim ENV version=2 -ENV PYTHONPATH /app +ENV PYTHONPATH=/app WORKDIR /app From ed57c15387f4f3e4087c0ed53f06c5ce3f8095d2 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sun, 3 Nov 2024 17:36:59 +0000 Subject: [PATCH 07/34] Use `__main__` instead of `sanic` command to avoid docker incompat --- nise-circleguard/Dockerfile | 7 +------ nise-circleguard/src/main.py | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/nise-circleguard/Dockerfile b/nise-circleguard/Dockerfile index 93ba737..07a656c 100644 --- a/nise-circleguard/Dockerfile +++ b/nise-circleguard/Dockerfile @@ -5,8 +5,6 @@ ENV PYTHONPATH=/app WORKDIR /app -RUN apt update - COPY requirements.txt ./requirements.txt RUN pip3 install --no-cache-dir -r requirements.txt @@ -21,9 +19,6 @@ RUN sed -i '238s|return \[x for x in arr if lower_limit < x < upper_limit\]|arr_ COPY ./src/ ./src/ -ENV GUNICORN_CMD_ARGS="--bind=0.0.0.0:5000 --workers=16" - WORKDIR /app/src -# Run gunicorn with the application -CMD ["sanic", "main", "--port=5000"] +CMD ["python", "main.py"] diff --git a/nise-circleguard/src/main.py b/nise-circleguard/src/main.py index faeaf15..f27ad24 100644 --- a/nise-circleguard/src/main.py +++ b/nise-circleguard/src/main.py @@ -291,3 +291,6 @@ async def process_similarity(request: Request): except ValueError as e: raise exceptions.BadRequest(str(e)) + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000) \ No newline at end of file From 649754166e4758f05e82f27478f621ebbbb34693 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sun, 3 Nov 2024 17:37:07 +0000 Subject: [PATCH 08/34] Update cg to new version --- nise-circleguard/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-circleguard/requirements.txt b/nise-circleguard/requirements.txt index a070b01..d0503bf 100644 --- a/nise-circleguard/requirements.txt +++ b/nise-circleguard/requirements.txt @@ -1,4 +1,4 @@ ossapi==3.4.3 -circleguard==5.4.1 +circleguard==5.4.2 brparser==1.0.4 sanic==24.6.0 From 1b4e0b52bf6350baf850938e0bdab61c12825247 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Tue, 5 Nov 2024 20:02:01 +0000 Subject: [PATCH 09/34] Update frontend environment to prep for release --- nise-frontend/src/environments/environment.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nise-frontend/src/environments/environment.ts b/nise-frontend/src/environments/environment.ts index d01eca8..e07433a 100644 --- a/nise-frontend/src/environments/environment.ts +++ b/nise-frontend/src/environments/environment.ts @@ -1,5 +1,5 @@ export const environment = { production: true, - apiUrl: 'https://nise.moe/api', - wsUrl: 'wss://nise.moe/api/websocket', + apiUrl: 'https://nise.stedos.dev/api', + wsUrl: 'wss://nise.stedos.dev/api/websocket', }; From 750de4ef096c453998b6da61afea7cca3e40f34f Mon Sep 17 00:00:00 2001 From: Stedoss Date: Tue, 5 Nov 2024 20:26:51 +0000 Subject: [PATCH 10/34] Update frontend to remove references to nise.moe --- nise-frontend/src/app/app-routing.module.ts | 2 +- nise-frontend/src/app/app.component.html | 4 ++-- nise-frontend/src/app/home/home.component.html | 4 ++-- nise-frontend/src/app/text-report.service.ts | 6 +++++- .../app/view-replay-pair/view-replay-pair.component.html | 4 ++-- .../src/app/view-score/view-score.component.html | 4 ++-- nise-frontend/src/environments/environment.ts | 7 +++++-- nise-frontend/src/index.html | 8 ++++---- 8 files changed, 23 insertions(+), 16 deletions(-) diff --git a/nise-frontend/src/app/app-routing.module.ts b/nise-frontend/src/app/app-routing.module.ts index 4b2d303..85f745b 100644 --- a/nise-frontend/src/app/app-routing.module.ts +++ b/nise-frontend/src/app/app-routing.module.ts @@ -39,7 +39,7 @@ const routes: Routes = [ {path: 'neko', component: MetabaseComponent, title: 'metabase integration'}, - {path: '**', component: HomeComponent, title: '/nise.moe/'}, + {path: '**', component: HomeComponent, title: '/nise.stedos.dev/'}, ]; @NgModule({ diff --git a/nise-frontend/src/app/app.component.html b/nise-frontend/src/app/app.component.html index c1f0f17..185ddcf 100644 --- a/nise-frontend/src/app/app.component.html +++ b/nise-frontend/src/app/app.component.html @@ -5,7 +5,7 @@
-

/nise.moe/

+

/nise.stedos.dev/

From 2d545a8c8217b4eec1a80d843ce347f00bff2f5c Mon Sep 17 00:00:00 2001 From: Stedoss Date: Tue, 5 Nov 2024 20:35:38 +0000 Subject: [PATCH 14/34] Fix header styling with new, longer link --- nise-frontend/src/assets/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-frontend/src/assets/style.css b/nise-frontend/src/assets/style.css index 6d19b17..7cc4723 100644 --- a/nise-frontend/src/assets/style.css +++ b/nise-frontend/src/assets/style.css @@ -69,7 +69,7 @@ html { } .header { - width: 555px; + width: 600px; text-align: center; } From b4440528ccfb0cecae3e0478f6b118996b2899e0 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Wed, 6 Nov 2024 18:01:38 +0000 Subject: [PATCH 15/34] Update nginx configs to use new domain --- nise-infra/nginx-main.conf | 4 +-- nise-infra/nise-data/nginx.conf | 58 ++------------------------------- 2 files changed, 4 insertions(+), 58 deletions(-) diff --git a/nise-infra/nginx-main.conf b/nise-infra/nginx-main.conf index 9ff4d95..d6cf33e 100644 --- a/nise-infra/nginx-main.conf +++ b/nise-infra/nginx-main.conf @@ -9,13 +9,13 @@ http { # Redirect HTTP to HTTPS server { listen 80; - server_name nise.moe replay.nise.moe neko.nise.moe; + server_name nise.stedos.dev; return 301 https://$host$request_uri; } server { listen 443 ssl; - server_name nise.moe replay.nise.moe git.nise.moe neko.nise.moe; + server_name nise.stedos.dev; ssl_certificate /etc/ssl/certs/nisemoe/certificate.pem; ssl_certificate_key /etc/ssl/certs/nisemoe/private.key; diff --git a/nise-infra/nise-data/nginx.conf b/nise-infra/nise-data/nginx.conf index dd5807f..d4636c2 100644 --- a/nise-infra/nise-data/nginx.conf +++ b/nise-infra/nise-data/nginx.conf @@ -2,29 +2,17 @@ events {} http { - upstream gitea { - server gitea:3000; - } - upstream nise-frontend { - server nise-frontend2:80; - } - - upstream nise-replay-viewer { - server nise-replay-viewer:80; + server nise-frontend:80; } upstream nise-backend { server nise-backend:8080; } - upstream nise-metabase { - server nise-metabase:3000; - } - server { listen 80; - server_name nise.moe; + server_name nise.stedos.dev; location / { proxy_pass http://nise-frontend; @@ -49,46 +37,4 @@ http { } - server { - listen 80; - - server_name git.nise.moe; - - location / { - client_max_body_size 10G; - proxy_pass http://gitea/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - - } - - server { - listen 80; - server_name replay.nise.moe; - - location / { - proxy_pass http://nise-replay-viewer/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - } - - server { - listen 80; - server_name neko.nise.moe; - - location / { - proxy_pass http://nise-metabase/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - } - } From 2a0b290211807ac7de6e742ec04db3ea67fa2eba Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Fri, 15 Nov 2024 22:33:32 +0000 Subject: [PATCH 16/34] Set `groupData` and `showPercentages` to false by default on graph views --- .../chart-hit-distribution.component.ts | 4 ++-- nise-frontend/src/corelib/components/chart/chart.component.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nise-frontend/src/corelib/components/chart-hit-distribution/chart-hit-distribution.component.ts b/nise-frontend/src/corelib/components/chart-hit-distribution/chart-hit-distribution.component.ts index 7ad7c73..107ffdf 100644 --- a/nise-frontend/src/corelib/components/chart-hit-distribution/chart-hit-distribution.component.ts +++ b/nise-frontend/src/corelib/components/chart-hit-distribution/chart-hit-distribution.component.ts @@ -23,8 +23,8 @@ export class ChartHitDistributionComponent implements OnInit, OnChanges { @Input() mods!: string[]; removeOutliers = true; - groupData = true; - showPercentages = true; + groupData = false; + showPercentages = false; public barChartLegend = true; public barChartPlugins = []; diff --git a/nise-frontend/src/corelib/components/chart/chart.component.ts b/nise-frontend/src/corelib/components/chart/chart.component.ts index d7e05f1..dda9073 100644 --- a/nise-frontend/src/corelib/components/chart/chart.component.ts +++ b/nise-frontend/src/corelib/components/chart/chart.component.ts @@ -43,8 +43,8 @@ export class ChartComponent implements OnChanges { @Input() data!: number[]; removeOutliers = true; - groupData = true; - showPercentages = true; + groupData = false; + showPercentages = false; calculateStatistics(): Array<{ name: string, value: number }> { if (this.data.length === 0) { From 7886e509dd111299cef01c530f1b0233dd760eeb Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sat, 16 Nov 2024 15:16:13 +0000 Subject: [PATCH 17/34] Allow query param value to be null for better code quality --- nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt index af95d33..4112706 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt @@ -70,14 +70,15 @@ class OsuApi( .version(HttpClient.Version.HTTP_2) .build() - fun doRequest(url: String, queryParams: Map, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse? { + fun doRequest(url: String, queryParams: Map, authorized: Boolean = true, appendToUrl: String? = null): HttpResponse? { var accessToken: TokenService.AccessTokenResponse? = null if(authorized) accessToken = this.tokenService.getAccessToken() val uriBuilder = StringBuilder(url) queryParams.forEach { (key, value) -> - uriBuilder.append("$key=$value&") + if (value != null) + uriBuilder.append("$key=$value&") } if(appendToUrl != null) From e06b5f3c7cb4235b8ef35964f4d76cbd76750f04 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sat, 16 Nov 2024 15:46:14 +0000 Subject: [PATCH 18/34] Implement `getUserMostPlayed` from osu!api --- .../main/kotlin/com/nisemoe/nise/osu/OsuApi.kt | 18 ++++++++++++++++++ .../com/nisemoe/nise/osu/OsuApiModels.kt | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt index 4112706..6291213 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt @@ -294,6 +294,24 @@ class OsuApi( } } + fun getUserMostPlayed(userId: Int, limit: Int? = null, offset: Int? = null): List? { + val queryParams = mapOf( + "limit" to limit, + "offset" to offset, + ) + val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/beatmapsets/most_played/?", queryParams) + + if (response == null) { + this.logger.info("Error getting user most played ($userId)") + return null + } + + return when (response.statusCode()) { + 200 -> serializer.decodeFromString>(response.body()) + else -> null + } + } + var rateLimitRemaining: Long = 0L var rateLimitTotal: Long = 0L diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt index 095a3b4..903f1cc 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt @@ -232,4 +232,10 @@ class OsuApiModels { val content: String ) + @Serializable + data class BeatmapPlaycount( + val beatmap_id: Int, + val count: Int, + ) + } \ No newline at end of file From aea087af64a4856f75b5d9e0f0a3d62e91a606ba Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sat, 16 Nov 2024 16:51:15 +0000 Subject: [PATCH 19/34] Add required field for User --- .../src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt index 903f1cc..1599a8d 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt @@ -39,6 +39,7 @@ class OsuApiModels { val avatar_url: String, val id: Long, val username: String, + val beatmap_playcounts_count: Int?, // Documentation: https://osu.ppy.sh/docs/index.html#userextended val join_date: String?, From e0cabfefcf3045751c58078bab2c79700838e7f5 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sun, 17 Nov 2024 00:26:31 +0000 Subject: [PATCH 20/34] Implement `getUserBeatmapScores` --- .../main/kotlin/com/nisemoe/nise/osu/OsuApi.kt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt index 6291213..0346bfb 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt @@ -214,6 +214,20 @@ class OsuApi( } } + fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? { + val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", mapOf()) + + if(response == null) { + this.logger.info("Error getting scores on beatmap $beatmapId for user $userId") + return null + } + + return when (response.statusCode()) { + 200 -> serializer.decodeFromString(response.body()) + else -> null + } + } + fun searchBeatmapsets(cursor: OsuApiModels.BeatmapsetSearchResultCursor?): OsuApiModels.BeatmapsetSearchResult? { val queryParams = mutableMapOf( "s" to "ranked", // Status [only ranked] @@ -294,7 +308,7 @@ class OsuApi( } } - fun getUserMostPlayed(userId: Int, limit: Int? = null, offset: Int? = null): List? { + fun getUserMostPlayed(userId: Long, limit: Int? = null, offset: Int? = null): List? { val queryParams = mapOf( "limit" to limit, "offset" to offset, From 7d44e4014b6c2a257b674758f86681e9111203e5 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Mon, 18 Nov 2024 01:30:56 +0000 Subject: [PATCH 21/34] Add ability to get a beatmap from osu!api from ID --- .../main/kotlin/com/nisemoe/nise/osu/OsuApi.kt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt index 0346bfb..87f2244 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt @@ -137,6 +137,19 @@ class OsuApi( } } + fun getBeatmapFromId(beatmapId: Int): OsuApiModels.Beatmap? { + val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId", emptyMap()) + if (response == null) { + this.logger.info("Error loading beatmap $beatmapId") + return null + } + + return when (response.statusCode()) { + 200 -> serializer.decodeFromString(response.body()) + else -> null + } + } + /** * Retrieves the replay data for a given score ID from the osu!api. * Efficiently cycles through the API keys to avoid rate limiting. @@ -215,7 +228,7 @@ class OsuApi( } fun getUserBeatmapScores(userId: Long, beatmapId: Int): OsuApiModels.BeatmapScores? { - val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", mapOf()) + val response = doRequest("https://osu.ppy.sh/api/v2/beatmaps/$beatmapId/scores/users/$userId/all", emptyMap()) if(response == null) { this.logger.info("Error getting scores on beatmap $beatmapId for user $userId") @@ -249,7 +262,7 @@ class OsuApi( } fun checkIfUserBanned(userId: Long): Boolean? { - val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", mapOf()) + val response = this.doRequest("https://osu.ppy.sh/api/v2/users/$userId/osu?key=id", emptyMap()) if(response == null) { this.logger.info("Error loading user with userId = $userId") return null From 31f301eab2615793c513b01d248a032a9a3d67ee Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Mon, 18 Nov 2024 01:32:20 +0000 Subject: [PATCH 22/34] Allow data class conversions from `Beatmap` to `ScoreBeatmap` --- .../com/nisemoe/nise/osu/OsuApiModels.kt | 2 ++ .../nise/osu/OsuApiModelsExtensions.kt | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt index 1599a8d..cd4a613 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModels.kt @@ -205,6 +205,7 @@ class OsuApiModels { data class Beatmap( val beatmapset_id: Int, val difficulty_rating: Double?, + val checksum: String?, val id: Int, val version: String?, val beatmapset: BeatmapSet, @@ -222,6 +223,7 @@ class OsuApiModels { @Serializable data class BeatmapSet( + val id: Int, val artist: String?, val creator: String?, val source: String?, diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt new file mode 100644 index 0000000..de0e9af --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApiModelsExtensions.kt @@ -0,0 +1,28 @@ +package com.nisemoe.nise.osu + +fun OsuApiModels.Beatmap.toScoreBeatmap(): OsuApiModels.ScoreBeatmap = + OsuApiModels.ScoreBeatmap( + id = this.id, + checksum = this.checksum, + difficulty_rating = this.difficulty_rating, + version = this.version, + max_combo = this.max_combo, + total_length = this.total_length, + bpm = this.bpm, + accuracy = this.accuracy, + ar = this.ar, + cs = this.cs, + drain = this.drain, + count_circles = this.count_circles, + count_sliders = this.count_sliders, + count_spinners = this.count_spinners, + ) + +fun OsuApiModels.BeatmapSet.toScoreBeatmapSet(): OsuApiModels.ScoreBeatmapset = + OsuApiModels.ScoreBeatmapset( + id = this.id, + title = this.title, + artist = this.artist, + creator = this.creator, + source = this.source, + ) \ No newline at end of file From dc846854e4b350f979e63ab79d2a4bb892cf16e0 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Mon, 18 Nov 2024 01:42:50 +0000 Subject: [PATCH 23/34] Processing a user in the queue now downloads all available replays (best effort) --- .../nisemoe/nise/scheduler/ImportScores.kt | 110 +++++++++++++----- 1 file changed, 78 insertions(+), 32 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index 1c60490..31e2d84 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -13,9 +13,7 @@ import com.nisemoe.nise.integrations.DiscordService import com.nisemoe.nise.konata.Replay import com.nisemoe.nise.konata.ReplaySetComparison import com.nisemoe.nise.konata.compareReplaySet -import com.nisemoe.nise.osu.Mod -import com.nisemoe.nise.osu.OsuApi -import com.nisemoe.nise.osu.OsuApiModels +import com.nisemoe.nise.osu.* import com.nisemoe.nise.service.CacheService import com.nisemoe.nise.service.CompressReplay import com.nisemoe.nise.service.UpdateUserQueueService @@ -165,38 +163,86 @@ class ImportScores( } for(userId in queue) { - val topUserScores = this.osuApi.getTopUserScores(userId = userId) - val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent") - val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts") + val user = this.osuApi.getUserProfile(userId.toString()) + + if (user == null) { + this.logger.error("Failed to fetch user from queue $userId") + this.updateUserQueueService.setUserAsProcessed(userId, failed = true) + continue; + } + + var userScores = mutableListOf() + + if (user.beatmap_playcounts_count != null) { + val mapsPlayed: MutableSet = mutableSetOf() + + this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays") + + for (page in 1..(user.beatmap_playcounts_count / 50) + 1) { + val maps = this.osuApi.getUserMostPlayed(userId, 50, 50 * page) + ?: break + + mapsPlayed.addAll(maps.map { it.beatmap_id }) + + this.logger.info("Page: $page/${(user.beatmap_playcounts_count / 50) + 1}") + + Thread.sleep(SLEEP_AFTER_API_CALL) + } + + var scoreProcessCount = 0 + for (mapId in mapsPlayed) { + val scores = this.osuApi.getUserBeatmapScores(userId, mapId) + ?: continue + + for (mapScore in scores.scores) { + if (mapScore.replay && mapScore.id != null) { + val beatmap = this.osuApi.getBeatmapFromId(mapId) + ?: continue + + userScores.add(mapScore.copy( + beatmap = beatmap.toScoreBeatmap(), + beatmapset = beatmap.beatmapset.toScoreBeatmapSet(), + )) + } + } + + this.logger.info( + "Getting all user scores for $userId: Processed map scores ${++scoreProcessCount}/${mapsPlayed.size}" + ) + + Thread.sleep(SLEEP_AFTER_API_CALL) + } + + } else { + val topUserScores = this.osuApi.getTopUserScores(userId = userId) + val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent") + val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts") + + if (topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { + this.logger.error("Failed to fetch top scores for user with id = $userId") + this.updateUserQueueService.setUserAsProcessed(userId, failed = true) + continue + } + + userScores += (topUserScores + recentUserScores + firstPlaceUserScores) + + userScores = userScores + .filter { it.beatmap != null && it.beatmapset != null } + .distinctBy { it.best_id } + .toMutableList() + } this.logger.info("Processing user with id = $userId") - this.logger.info("Top scores: ${topUserScores?.size}") - this.logger.info("Recent scores: ${recentUserScores?.size}") - this.logger.info("First place scores: ${firstPlaceUserScores?.size}") + this.logger.info("User has ${userScores.size} total scores") Thread.sleep(SLEEP_AFTER_API_CALL) - if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { - this.logger.error("Failed to fetch top scores for user with id = $userId") - this.updateUserQueueService.setUserAsProcessed(userId, failed = true) - continue - } - - val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores) - .filter { it.beatmap != null && it.beatmapset != null } - .distinctBy { it.best_id } - - this.logger.info("Unique scores: ${allUserScores.size}") + this.logger.info("Unique scores: ${userScores.size}") val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS))) - if(!userExists) { - val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id") - if(apiUser != null) { - this.userService.insertApiUser(apiUser) - this.statistics.usersAddedToDatabase++ - } else { - this.logger.error("Failed to fetch user with id = $userId") - } + if (!userExists) { + this.userService.insertApiUser(user) + this.statistics.usersAddedToDatabase++ } var current = 0 @@ -209,7 +255,7 @@ class ImportScores( .limit(1) .fetchOneInto(OffsetDateTime::class.java) - for(topScore in allUserScores) { + for(topScore in userScores) { val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id)) if (!beatmapExists) { val beatmapFile = this.osuApi.getBeatmapFile(beatmapId = topScore.beatmap.id) @@ -264,7 +310,7 @@ class ImportScores( // Update the database dslContext.update(UPDATE_USER_QUEUE) .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current) - .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size) + .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, userScores.size) .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) .and(UPDATE_USER_QUEUE.PROCESSED.isFalse) .execute() @@ -274,7 +320,7 @@ class ImportScores( lastCompletedUpdate = lastCompletedUpdate, canUpdate = false, progressCurrent = current, - progressTotal = allUserScores.size + progressTotal = userScores.size ) // Update the frontend @@ -289,7 +335,7 @@ class ImportScores( } // Check for stolen replays. - val uniqueBeatmapIds = allUserScores + val uniqueBeatmapIds = userScores .groupBy { it.beatmap!!.id } this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.") From 02b41bae0d74fbe755b99fe95fb8634306e1ebb2 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Mon, 18 Nov 2024 02:03:53 +0000 Subject: [PATCH 24/34] Add guard to prevent mass score fetching of users we don't really care about --- .../kotlin/com/nisemoe/nise/scheduler/ImportScores.kt | 10 ++++++++-- .../com/nisemoe/nise/service/UpdateUserQueueService.kt | 7 +++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index 31e2d84..f25e799 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -162,7 +162,13 @@ class ImportScores( this.logger.info("Processing ${queue.size} users from the queue.") } - for(userId in queue) { + for(queueEntry in queue) { + val userId = queueEntry.userId + + // We should only 'full fetch' a user if they have been explicitly added by another user, + // else we will spend way too much time on random users. + val shouldFullFetch = queueEntry.addedByUserId != null + val user = this.osuApi.getUserProfile(userId.toString()) if (user == null) { @@ -173,7 +179,7 @@ class ImportScores( var userScores = mutableListOf() - if (user.beatmap_playcounts_count != null) { + if (shouldFullFetch && user.beatmap_playcounts_count != null) { val mapsPlayed: MutableSet = mutableSetOf() this.logger.info("User has ${user.beatmap_playcounts_count} unique beatmap plays") diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt index 480d696..e188b07 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt @@ -79,12 +79,11 @@ class UpdateUserQueueService( /** * Retrieves the full update queue, only pending users. */ - fun getQueue(): List { - return dslContext.select(UPDATE_USER_QUEUE.USER_ID) - .from(UPDATE_USER_QUEUE) + fun getQueue(): List { + return dslContext.selectFrom(UPDATE_USER_QUEUE) .where(UPDATE_USER_QUEUE.PROCESSED.isFalse) .orderBy(UPDATE_USER_QUEUE.CREATED_AT.asc()) - .fetchInto(Long::class.java) + .fetch() } /** From d09d3c8c7719ef5ce3c6b7c006a7e0a4efc59275 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:29:19 +0000 Subject: [PATCH 25/34] First attempt at adding caddy --- nise-infra/Caddyfile | 3 +++ nise-infra/docker-compose.yml | 11 ++++------- 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 nise-infra/Caddyfile diff --git a/nise-infra/Caddyfile b/nise-infra/Caddyfile new file mode 100644 index 0000000..5df8317 --- /dev/null +++ b/nise-infra/Caddyfile @@ -0,0 +1,3 @@ +nise.stedos.dev { + reverse_proxy nise-nginx +} \ No newline at end of file diff --git a/nise-infra/docker-compose.yml b/nise-infra/docker-compose.yml index d5642a8..25975d8 100644 --- a/nise-infra/docker-compose.yml +++ b/nise-infra/docker-compose.yml @@ -2,15 +2,12 @@ version: '3' services: - nginx-main: - image: nginx:latest - container_name: nginx-main + caddy-main: + image: caddy:alpine + container_name: caddy-main restart: always volumes: - - ./nginx-main.conf:/etc/nginx/nginx.conf:ro - # nise.moe certificates (by Cloudflare) - - ./nise-data/certificate.pem:/etc/ssl/certs/nisemoe/certificate.pem:ro - - ./nise-data/private.key:/etc/ssl/certs/nisemoe/private.key:ro + - ./Caddyfile:/etc/caddy/Caddyfile:ro ports: - "443:443" - "80:80" From e0a7cbbfcbe8f44b4fbef049b7e87092f9ade565 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:48:19 +0000 Subject: [PATCH 26/34] Sync main docker compose --- nise-infra/docker-compose.yml | 131 ++++++++++------------------------ 1 file changed, 37 insertions(+), 94 deletions(-) diff --git a/nise-infra/docker-compose.yml b/nise-infra/docker-compose.yml index 25975d8..1816efc 100644 --- a/nise-infra/docker-compose.yml +++ b/nise-infra/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: caddy-main: @@ -11,10 +9,17 @@ services: ports: - "443:443" - "80:80" + depends_on: + - nise-nginx # Shared services which are used by others + redis: + image: redis:alpine + container_name: redis + restart: always + postgres: - image: groonga/pgroonga:3.1.6-alpine-15 + image: postgres:alpine container_name: postgres restart: always environment: @@ -22,47 +27,6 @@ services: POSTGRES_PASSWORD: ${DB_PASS} volumes: - postgres-data:/var/lib/postgresql/data - command: > - -c shared_buffers=6GB - -c effective_cache_size=12GB - -c work_mem=64MB - -c maintenance_work_mem=2GB - -c checkpoint_completion_target=0.9 - -c checkpoint_timeout=15min - -c max_wal_size=2GB - -c wal_buffers=16MB - -c max_connections=100 - -c max_worker_processes=8 - -c max_parallel_workers_per_gather=4 - -c max_parallel_workers=8 - -c effective_io_concurrency=40 - shm_size: '128mb' - - redis: - image: redis:alpine - container_name: redis - restart: always - - # ------------------------------------------------------------------ - - gitea: - image: gitea/gitea - container_name: gitea - restart: always - environment: - USER_UID: 1336 - USER_GID: 1336 - GITEA__database__DB_TYPE: postgres - GITEA__database__HOST: ${DB_HOST}:5432 - GITEA__database__NAME: gitea - GITEA__database__USER: ${DB_USER} - GITEA__database__PASSWD: ${DB_PASS} - depends_on: - - postgres - - redis - volumes: - - ./gitea-data/app.ini:/data/gitea/conf/app.ini - - gitea-data:/data # ------------------------------------------------------------------ @@ -72,19 +36,31 @@ services: restart: always volumes: - ./nise-data/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - nise-backend + - nise-frontend + + nise-circleguard: + image: code.stedos.dev/stedos/nise-circleguard:latest + container_name: nise-circleguard + environment: + OSU_API_KEY: ${OSU_API_KEY} + restart: always + volumes: + - ./nise-data/beatmaps:/app/dbs nise-backend: - image: git.nise.moe/nuff/nise-backend:latest + image: code.stedos.dev/stedos/nise-backend:latest container_name: nise-backend environment: - SPRING_PROFILES_ACTIVE: postgres,discord,import:scores,import:users,fix:scores + SPRING_PROFILES_ACTIVE: postgres,import:scores,import:users,fix:scores # App configuration OLD_SCORES_PAGE_SIZE: 1000 # Postgres POSTGRES_HOST: ${DB_HOST} POSTGRES_USER: ${DB_USER} POSTGRES_PASS: ${DB_PASS} - POSTGRES_DB: nise + POSTGRES_DB: ${DB_NAME} # redis REDIS_DB: 4 # Discord @@ -94,69 +70,36 @@ services: OSU_API_KEY: ${OSU_API_KEY} OSU_CLIENT_ID: ${OSU_CLIENT_ID} OSU_CLIENT_SECRET: ${OSU_CLIENT_SECRET} - OSU_CALLBACK: "https://nise.moe/api/login/oauth2/code/osu" + OSU_CALLBACK: "https://nise.stedos.dev/api/login/oauth2/code/osu" # Metabase METABASE_API_KEY: ${METABASE_API_KEY} # Internal API CIRCLEGUARD_API_URL: http://nise-circleguard:5000 # Auth - ORIGIN: "https://nise.moe" + ORIGIN: "https://nise.stedos.dev" REPLAY_ORIGIN: "https://replay.nise.moe" COOKIE_SECURE: false BEATMAPS_PATH: "/app/dbs" + # Replay cache + REPLAY_CACHE_ENABLED: ${REPLAY_CACHE_ENABLED} + REPLAY_CACHE_HOST: ${REPLAY_CACHE_HOST} + REPLAY_CACHE_PORT: ${REPLAY_CACHE_PORT} + REPLAY_CACHE_DB: ${REPLAY_CACHE_DB} + REPLAY_CACHE_USER: ${REPLAY_CACHE_USER} + REPLAY_CACHE_PASS: ${REPLAY_CACHE_PASS} restart: always volumes: - ./nise-data/beatmaps:/app/dbs depends_on: - postgres - redis + - nise-circleguard - nise-circleguard: - image: git.nise.moe/nuff/nise-circleguard:latest - container_name: nise-circleguard - environment: - OSU_API_KEY: ${OSU_API_KEY} - restart: always - volumes: - - ./nise-data/beatmaps:/app/dbs - - nise-frontend2: - image: git.nise.moe/nuff/nise-frontend:latest - container_name: nise-frontend2 + nise-frontend: + image: code.stedos.dev/stedos/nise-frontend:latest + container_name: nise-frontend restart: always - nise-replay-viewer: - image: git.nise.moe/nuff/nise-replay-viewer:latest - container_name: nise-replay-viewer - restart: always - - nise-discord: - image: git.nise.moe/nuff/nise-discord:latest - container_name: nise-discord - environment: - DISCORD_TOKEN: ${DISCORD_TOKEN} - REACTION_CHANNEL_ID: ${REACTION_CHANNEL_ID} - REACTION_EMOJI_ID: ${REACTION_EMOJI_ID} - restart: always - - nise-metabase: - image: metabase/metabase:latest - container_name: nise-metabase - volumes: - - /dev/urandom:/dev/random:ro - environment: - MB_DB_TYPE: postgres - MB_DB_DBNAME: metabase - MB_DB_PORT: 5432 - MB_DB_USER: ${DB_METABASE_USER} - MB_DB_PASS: ${DB_METABASE_PASS} - MB_DB_HOST: postgres - healthcheck: - test: curl --fail -I http://localhost:3000/api/health || exit 1 - interval: 15s - timeout: 5s - retries: 5 volumes: - postgres-data: - gitea-data: + postgres-data: \ No newline at end of file From 69829518a133565894003440aab05d34494e7ddc Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sat, 18 Jan 2025 23:48:37 +0000 Subject: [PATCH 27/34] Make text reporting more Reddit friendly --- nise-frontend/src/app/text-report.service.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nise-frontend/src/app/text-report.service.ts b/nise-frontend/src/app/text-report.service.ts index ec816dc..f4ca73e 100644 --- a/nise-frontend/src/app/text-report.service.ts +++ b/nise-frontend/src/app/text-report.service.ts @@ -25,26 +25,29 @@ export class TextReportService { report += `Profile: https://osu.ppy.sh/users/${userDetails.user_id}\n`; for (const suspiciousScore of suspiciousScores) { - report += `\n${this.getRelaxReport(suspiciousScore)}\n`; + report += `\n\n${this.getRelaxReport(suspiciousScore)}\n`; } for (const similarReplay of similarReplays) { - report += `\n${this.getStealingReport(similarReplay)}\n`; + report += `\n\n${this.getStealingReport(similarReplay)}\n`; } - report += `\nGenerated on ${site} - [${userDetails.username} on ${site}](${environment.webUrl}/u/${userDetails.user_id})`; + report += `\n\nGenerated on ${site} - [${userDetails.username} on ${site}](${environment.webUrl}/u/${userDetails.user_id})`; return report; } private static getRelaxReport(suspiciousScore: SuspiciousScore): string { return `[Replay on ${suspiciousScore.beatmap_title}](https://osu.ppy.sh/scores/osu/${suspiciousScore.replay_id}) + cvUR: ${suspiciousScore.ur.toFixed(2)} according to Circleguard`; } private static getStealingReport(similarReplay: SimilarReplay): string { return `[${similarReplay.username_2}'s replay (cheated)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_2}) + [${similarReplay.username_1}'s replay (original)](https://osu.ppy.sh/scores/osu/${similarReplay.replay_id_1}) + ${similarReplay.similarity.toFixed(2)} similarity according to Circleguard`; } } From eb7f9e1e2333a2c42596ae34e88042036a286973 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Sat, 18 Jan 2025 23:49:54 +0000 Subject: [PATCH 28/34] Bump frontend version --- nise-frontend/src/app/app.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-frontend/src/app/app.component.html b/nise-frontend/src/app/app.component.html index 5f81440..fa089c5 100644 --- a/nise-frontend/src/app/app.component.html +++ b/nise-frontend/src/app/app.component.html @@ -34,5 +34,5 @@
- v20241105 + v20250118
From 559124460dc83fbcc974411f3ba402b38825bf98 Mon Sep 17 00:00:00 2001 From: Stedoss Date: Sat, 18 Jan 2025 23:59:30 +0000 Subject: [PATCH 29/34] Use alpine nginx image for frontend container --- nise-frontend/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-frontend/Dockerfile b/nise-frontend/Dockerfile index 8216513..f43624d 100644 --- a/nise-frontend/Dockerfile +++ b/nise-frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.27.0 +FROM nginx:1.27.0-alpine RUN rm -rf /usr/share/nginx/html/* From aadada20845d5fc0acbe23f2a702ea4d68d0e100 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Thu, 13 Feb 2025 00:08:37 +0000 Subject: [PATCH 30/34] Add basic health check endpoint --- .../nise/controller/HealthController.kt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/controller/HealthController.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/HealthController.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/HealthController.kt new file mode 100644 index 0000000..47e6f78 --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/HealthController.kt @@ -0,0 +1,22 @@ +package com.nisemoe.nise.controller + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RestController + +data class HealthResponse( + val healthy: Boolean, +) + +val healthResponse = HealthResponse( + healthy = true, +) + +@RestController +class HealthController { + @GetMapping("/health") + fun healthCheck(): ResponseEntity { + return ResponseEntity.ok(healthResponse) + } +} + From 9d33c64949f381ea0f698ac5a0c9009e2bc12722 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Thu, 13 Feb 2025 22:23:46 +0000 Subject: [PATCH 31/34] Suspicious scores are now returned when cvur is <35 --- .../src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt | 2 +- .../view-suspicious-scores.component.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt index 2f4e3b7..b1db96f 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt @@ -232,7 +232,7 @@ class ScoreService( } fun getDefaultCondition(): Condition { - return SCORES.UR.lessOrEqual(25.0) + return SCORES.UR.lessOrEqual(35.0) .and(SCORES.IS_BANNED.eq(false)) } diff --git a/nise-frontend/src/app/view-suspicious-scores/view-suspicious-scores.component.html b/nise-frontend/src/app/view-suspicious-scores/view-suspicious-scores.component.html index e0faf7f..1a994d9 100644 --- a/nise-frontend/src/app/view-suspicious-scores/view-suspicious-scores.component.html +++ b/nise-frontend/src/app/view-suspicious-scores/view-suspicious-scores.component.html @@ -1,7 +1,7 @@

/sus/ - Suspicious Scores

- This includes all replays with <25 cvUR. Low values can indicate cheating but always manually review users and + This includes all replays with <35 cvUR. Low values can indicate cheating but always manually review users and replays before making judgements.
@@ -24,7 +24,7 @@

- +`

From d0c4964caa16a4ef02d9e90ac5de4413fb8c1bf4 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Thu, 13 Feb 2025 22:23:58 +0000 Subject: [PATCH 32/34] Update FE version number --- nise-frontend/src/app/app.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise-frontend/src/app/app.component.html b/nise-frontend/src/app/app.component.html index fa089c5..daae079 100644 --- a/nise-frontend/src/app/app.component.html +++ b/nise-frontend/src/app/app.component.html @@ -34,5 +34,5 @@

- v20250118 + v20250213
From d6d5953a44731071c4a99cd11100d654e1e1437c Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Thu, 13 Feb 2025 22:39:23 +0000 Subject: [PATCH 33/34] Add version endpoint for backend --- .../nise/controller/VersionController.kt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/controller/VersionController.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/VersionController.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/VersionController.kt new file mode 100644 index 0000000..6af856e --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/VersionController.kt @@ -0,0 +1,19 @@ +package com.nisemoe.nise.controller + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RestController + +data class VersionResponse( + val version: String, +) + +val versionResponse = VersionResponse( + version = "v20250213", +) + +@RestController +class VersionController { + @GetMapping("/version") + fun getVersion(): ResponseEntity = ResponseEntity.ok(versionResponse) +} \ No newline at end of file From 792b2032554fcb8119f56f2f8b8c155f3daf9492 Mon Sep 17 00:00:00 2001 From: Stedoss <29103029+Stedoss@users.noreply.github.com> Date: Mon, 17 Feb 2025 07:46:20 +0000 Subject: [PATCH 34/34] Add `mods_bitwise` to score endpoints --- nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt | 1 + .../main/kotlin/com/nisemoe/nise/database/ScoreService.kt | 5 ++++- .../kotlin/com/nisemoe/nise/database/UserScoreService.kt | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt index 48a3bc5..6517d84 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt @@ -141,6 +141,7 @@ data class ReplayData( val beatmap_count_sliders: Int?, val beatmap_count_spinners: Int?, val score: Int, + val mods_bitwise: Int, val mods: List, val rank: String?, val ur: Double?, diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt index b1db96f..33d9bf3 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt @@ -177,6 +177,8 @@ class ScoreService( val hitDistribution = this.getHitDistribution(scoreId = result.get(SCORES.ID, Int::class.java)) val charts = this.getCharts(result) + val mods = result.get(SCORES.MODS, Int::class.java) + val replayData = ReplayData( replay_id = replayId, user_id = result.get(SCORES.USER_ID, Int::class.java), @@ -204,7 +206,8 @@ class ScoreService( ur = result.get(SCORES.UR, Double::class.java), adjusted_ur = result.get(SCORES.ADJUSTED_UR, Double::class.java), score = result.get(SCORES.SCORE, Int::class.java), - mods = Mod.parseModCombination(result.get(SCORES.MODS, Int::class.java)), + mods_bitwise = mods, + mods = Mod.parseModCombination(mods), rank = result.get(SCORES.RANK, String::class.java), snaps = result.get(SCORES.SNAPS, Int::class.java), hits = result.get(SCORES.EDGE_HITS, Int::class.java), diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt index 23ec651..b9a5cca 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt @@ -76,6 +76,8 @@ class UserScoreService( val hitDistribution = this.getHitDistribution(result.get(USER_SCORES.JUDGEMENTS, ByteArray::class.java)) val charts = this.scoreService.getCharts(result) + val mods = result.get(USER_SCORES.MODS, Int::class.java) + val replayData = ReplayData( replay_id = result.get(USER_SCORES.ONLINE_SCORE_ID, Long::class.java), username = result.get(USER_SCORES.PLAYER_NAME, String::class.java), @@ -100,7 +102,8 @@ class UserScoreService( ur = result.get(USER_SCORES.UR, Double::class.java), adjusted_ur = result.get(USER_SCORES.ADJUSTED_UR, Double::class.java), score = result.get(USER_SCORES.TOTAL_SCORE, Int::class.java), - mods = Mod.parseModCombination(result.get(USER_SCORES.MODS, Int::class.java)), + mods_bitwise = mods, + mods = Mod.parseModCombination(mods), snaps = result.get(USER_SCORES.SNAPS, Int::class.java), hits = result.get(USER_SCORES.EDGE_HITS, Int::class.java), perfect = result.get(USER_SCORES.PERFECT, Boolean::class.java),