diff --git a/nise-backend/pom.xml b/nise-backend/pom.xml index 15314e4..c4c6cf9 100644 --- a/nise-backend/pom.xml +++ b/nise-backend/pom.xml @@ -33,6 +33,12 @@ spring-boot-starter-security + + com.aayushatharva.brotli4j + brotli4j + 1.16.0 + + org.testcontainers diff --git a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/Scores.kt b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/Scores.kt index e597e93..25b80ba 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/Scores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/Scores.kt @@ -297,6 +297,11 @@ open class Scores( */ val SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED: TableField = createField(DSL.name("sliderend_release_standard_deviation_adjusted"), SQLDataType.DOUBLE, this, "") + /** + * The column public.scores.judgements. + */ + val JUDGEMENTS: TableField = createField(DSL.name("judgements"), SQLDataType.BLOB, this, "") + private constructor(alias: Name, aliased: Table?): this(alias, null, null, aliased, null) private constructor(alias: Name, aliased: Table?, parameters: Array?>?): this(alias, null, null, aliased, parameters) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/ScoresRecord.kt b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/ScoresRecord.kt index d7f8f59..3b66a2b 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/ScoresRecord.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/ScoresRecord.kt @@ -201,6 +201,10 @@ open class ScoresRecord private constructor() : UpdatableRecordImpl? = null, keypressesMedian: Double? = null, keypressesStandardDeviation: Double? = null, sliderendReleaseTimes: Array? = null, sliderendReleaseMedian: Double? = null, sliderendReleaseStandardDeviation: Double? = null, keypressesMedianAdjusted: Double? = null, keypressesStandardDeviationAdjusted: Double? = null, sliderendReleaseMedianAdjusted: Double? = null, sliderendReleaseStandardDeviationAdjusted: Double? = null): this() { + constructor(id: Int? = null, beatmapId: Int? = null, count_100: Int? = null, count_300: Int? = null, count_50: Int? = null, countMiss: Int? = null, date: LocalDateTime? = null, maxCombo: Int? = null, mods: Int? = null, perfect: Boolean? = null, pp: Double? = null, rank: String? = null, replayAvailable: Boolean? = null, replayId: Long? = null, score: Long? = null, userId: Long? = null, replay: ByteArray? = null, ur: Double? = null, frametime: Double? = null, edgeHits: Int? = null, snaps: Int? = null, isBanned: Boolean? = null, adjustedUr: Double? = null, meanError: Double? = null, errorVariance: Double? = null, errorStandardDeviation: Double? = null, minimumError: Double? = null, maximumError: Double? = null, errorRange: Double? = null, errorCoefficientOfVariation: Double? = null, errorKurtosis: Double? = null, errorSkewness: Double? = null, sentDiscordNotification: Boolean? = null, addedAt: OffsetDateTime? = null, version: Int? = null, keypressesTimes: Array? = null, keypressesMedian: Double? = null, keypressesStandardDeviation: Double? = null, sliderendReleaseTimes: Array? = null, sliderendReleaseMedian: Double? = null, sliderendReleaseStandardDeviation: Double? = null, keypressesMedianAdjusted: Double? = null, keypressesStandardDeviationAdjusted: Double? = null, sliderendReleaseMedianAdjusted: Double? = null, sliderendReleaseStandardDeviationAdjusted: Double? = null, judgements: ByteArray? = null): this() { this.id = id this.beatmapId = beatmapId this.count_100 = count_100 @@ -256,6 +260,7 @@ open class ScoresRecord private constructor() : UpdatableRecordImpl { - val judgements = dslContext.selectFrom(SCORES_JUDGEMENTS) - .where(SCORES_JUDGEMENTS.SCORE_ID.eq(scoreId)) - .fetchInto(ScoresJudgementsRecord::class.java) + val judgementsRecord = dslContext.select(SCORES.JUDGEMENTS) + .from(SCORES) + .where(SCORES.ID.eq(scoreId)) + .fetchOneInto(ScoresRecord::class.java) ?: return emptyMap() + + if(judgementsRecord.judgements == null) return emptyMap() + + val judgements = compressJudgements.deserialize(judgementsRecord.judgements!!) val errorDistribution = mutableMapOf>() var totalHits = 0 judgements.forEach { hit -> - val error = (hit.error!!.roundToInt() / 2) * 2 + val error = (hit.error.roundToInt() / 2) * 2 val judgementType = hit.type // Assuming this is how you get the judgement type - errorDistribution.getOrPut(error) { mutableMapOf("Miss" to 0, "300" to 0, "100" to 0, "50" to 0) } + errorDistribution.getOrPut(error) { mutableMapOf("MISS" to 0, "THREE_HUNDRED" to 0, "ONE_HUNDRED" to 0, "FIFTY" to 0) } .apply { this[judgementType.toString()] = this.getOrDefault(judgementType.toString(), 0) + 1 } @@ -387,10 +394,10 @@ class ScoreService( return errorDistribution.mapValues { (_, judgementCounts) -> judgementCounts.values.sum() DistributionEntry( - percentageMiss = (judgementCounts.getOrDefault("Miss", 0).toDouble() / totalHits) * 100, - percentage300 = (judgementCounts.getOrDefault("300", 0).toDouble() / totalHits) * 100, - percentage100 = (judgementCounts.getOrDefault("100", 0).toDouble() / totalHits) * 100, - percentage50 = (judgementCounts.getOrDefault("50", 0).toDouble() / totalHits) * 100 + percentageMiss = (judgementCounts.getOrDefault("MISS", 0).toDouble() / totalHits) * 100, + percentage300 = (judgementCounts.getOrDefault("THREE_HUNDRED", 0).toDouble() / totalHits) * 100, + percentage100 = (judgementCounts.getOrDefault("ONE_HUNDRED", 0).toDouble() / totalHits) * 100, + percentage50 = (judgementCounts.getOrDefault("FIFTY", 0).toDouble() / totalHits) * 100 ) } } diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt index 04ba504..782b83b 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt @@ -2,9 +2,8 @@ package com.nisemoe.nise.scheduler import com.nisemoe.generated.tables.records.ScoresRecord import com.nisemoe.generated.tables.references.SCORES -import com.nisemoe.generated.tables.references.SCORES_JUDGEMENTS import com.nisemoe.nise.integrations.CircleguardService -import com.nisemoe.nise.Format.Companion.fromJudgementType +import com.nisemoe.nise.service.CompressJudgements import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.joinAll @@ -16,23 +15,27 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Profile import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service -import java.time.OffsetDateTime @Profile("old_scores") @Service class FixOldScores( private val dslContext: DSLContext, - private val circleguardService: CircleguardService + private val circleguardService: CircleguardService, + private val compressJudgements: CompressJudgements ){ + companion object { + + const val CURRENT_VERSION = 6 + + } + @Value("\${OLD_SCORES_WORKERS:4}") private var workers: Int = 4 @Value("\${OLD_SCORES_PAGE_SIZE:5000}") private var pageSize: Int = 5000 - val CURRENT_VERSION = 5 - private val logger = LoggerFactory.getLogger(javaClass) data class Task(val offset: Int, val limit: Int) @@ -139,6 +142,7 @@ class FixOldScores( .set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted) + .set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements)) .where(SCORES.REPLAY_ID.eq(score.replayId)) .returningResult(SCORES.ID) .fetchOne()?.getValue(SCORES.ID) @@ -152,22 +156,6 @@ class FixOldScores( .set(SCORES.VERSION, CURRENT_VERSION) .where(SCORES.ID.eq(scoreId)) .execute() - - val judgementsExist = dslContext.fetchExists(SCORES_JUDGEMENTS, SCORES_JUDGEMENTS.SCORE_ID.eq(scoreId)) - if(!judgementsExist) { - for (judgement in processedReplay.judgements) { - dslContext.insertInto(SCORES_JUDGEMENTS) - .set(SCORES_JUDGEMENTS.TIME, judgement.time) - .set(SCORES_JUDGEMENTS.X, judgement.x) - .set(SCORES_JUDGEMENTS.Y, judgement.y) - .set(SCORES_JUDGEMENTS.TYPE, fromJudgementType(judgement.type)) - .set(SCORES_JUDGEMENTS.DISTANCE_EDGE, judgement.distance_edge) - .set(SCORES_JUDGEMENTS.DISTANCE_CENTER, judgement.distance_center) - .set(SCORES_JUDGEMENTS.ERROR, judgement.error) - .set(SCORES_JUDGEMENTS.SCORE_ID, scoreId) - .execute() - } - } } } \ No newline at end of file diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index e874b0d..9350266 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -5,7 +5,6 @@ import com.nisemoe.generated.tables.references.* import com.nisemoe.konata.Replay import com.nisemoe.konata.ReplaySetComparison import com.nisemoe.konata.compareReplaySet -import com.nisemoe.nise.Format.Companion.fromJudgementType import com.nisemoe.nise.UserQueueDetails import com.nisemoe.nise.database.ScoreService import com.nisemoe.nise.database.UserService @@ -16,6 +15,7 @@ import com.nisemoe.nise.osu.Mod import com.nisemoe.nise.osu.OsuApi import com.nisemoe.nise.osu.OsuApiModels import com.nisemoe.nise.service.CacheService +import com.nisemoe.nise.service.CompressJudgements import com.nisemoe.nise.service.UpdateUserQueueService import kotlinx.serialization.Serializable import org.jooq.DSLContext @@ -48,7 +48,8 @@ class ImportScores( private val scoreService: ScoreService, private val updateUserQueueService: UpdateUserQueueService, private val circleguardService: CircleguardService, - private val messagingTemplate: SimpMessagingTemplate + private val messagingTemplate: SimpMessagingTemplate, + private val compressJudgements: CompressJudgements ) : InitializingBean { private val userToUpdateBucket = mutableListOf() @@ -65,19 +66,22 @@ class ImportScores( this.cacheService.deleteVariable("userToUpdateBucket") } } - - val CURRENT_VERSION = 5 - + + companion object { + + const val CURRENT_VERSION = 6 + const val SLEEP_AFTER_API_CALL = 500L + const val UPDATE_USER_EVERY_DAYS = 7L + const val UPDATE_BANNED_USERS_EVERY_DAYS = 3L + + } + @Value("\${WEBHOOK_URL}") private lateinit var webhookUrl: String private val logger = LoggerFactory.getLogger(javaClass) - private final val sleepTimeInMs = 500L - - private final val UPDATE_USER_EVERY_DAYS = 7L - private final val UPDATE_BANNED_USERS_EVERY_DAYS = 3L - + data class UpdaterStatistics( var currentBeatmapsetPage: Int = 0, @@ -165,7 +169,7 @@ class ImportScores( this.logger.info("Recent scores: ${recentUserScores?.size}") this.logger.info("First place scores: ${firstPlaceUserScores?.size}") - Thread.sleep(this.sleepTimeInMs) + Thread.sleep(SLEEP_AFTER_API_CALL) if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { this.logger.error("Failed to fetch top scores for user with id = $userId") @@ -297,7 +301,7 @@ class ImportScores( .where(SCORES.USER_ID.eq(userId)) .execute() } - Thread.sleep(this.sleepTimeInMs) + Thread.sleep(SLEEP_AFTER_API_CALL) } this.cacheService.setVariable("lastBannedUserCheck", LocalDateTime.now()) @@ -313,11 +317,11 @@ class ImportScores( do { val searchResults = this.osuApi.searchBeatmapsets(cursor = cursor) this.statistics.currentBeatmapsetPage++ - Thread.sleep(this.sleepTimeInMs) + Thread.sleep(SLEEP_AFTER_API_CALL) if (searchResults == null) { this.logger.error("Failed to fetch beatmapsets. Skipping to next page...") - Thread.sleep(this.sleepTimeInMs * 2) + Thread.sleep(SLEEP_AFTER_API_CALL * 2) return } @@ -365,11 +369,11 @@ class ImportScores( } val beatmapScores = this.osuApi.getTopBeatmapScores(beatmapId = beatmap.id) - Thread.sleep(this.sleepTimeInMs) + Thread.sleep(SLEEP_AFTER_API_CALL) if (beatmapScores == null) { this.logger.error("Failed to fetch beatmap scores for beatmapId = ${beatmap.id}") - Thread.sleep(this.sleepTimeInMs * 2) + Thread.sleep(SLEEP_AFTER_API_CALL * 2) continue } @@ -394,7 +398,7 @@ class ImportScores( if(this.userToUpdateBucket.size >= 50) { val usersBucket = this.osuApi.getUsersBatch(this.userToUpdateBucket.toList()) - Thread.sleep(this.sleepTimeInMs) + Thread.sleep(SLEEP_AFTER_API_CALL) if(usersBucket == null) { this.logger.error("Failed to fetch users batch.") continue @@ -648,6 +652,7 @@ class ImportScores( .set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted) + .set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements)) .where(SCORES.REPLAY_ID.eq(score.best_id)) .returningResult(SCORES.ID) .fetchOne()?.getValue(SCORES.ID) @@ -683,19 +688,6 @@ class ImportScores( this.updateUserQueueService.insertUser(score.user_id) } } - - for (judgement in processedReplay.judgements) { - dslContext.insertInto(SCORES_JUDGEMENTS) - .set(SCORES_JUDGEMENTS.TIME, judgement.time) - .set(SCORES_JUDGEMENTS.X, judgement.x) - .set(SCORES_JUDGEMENTS.Y, judgement.y) - .set(SCORES_JUDGEMENTS.TYPE, fromJudgementType(judgement.type)) - .set(SCORES_JUDGEMENTS.DISTANCE_EDGE, judgement.distance_edge) - .set(SCORES_JUDGEMENTS.DISTANCE_CENTER, judgement.distance_center) - .set(SCORES_JUDGEMENTS.ERROR, judgement.error) - .set(SCORES_JUDGEMENTS.SCORE_ID, scoreId) - .execute() - } } } \ No newline at end of file diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt new file mode 100644 index 0000000..2ffad37 --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt @@ -0,0 +1,78 @@ +package com.nisemoe.nise.service + +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.Decoder +import com.aayushatharva.brotli4j.encoder.Encoder +import com.nisemoe.nise.integrations.CircleguardService +import org.springframework.stereotype.Service +import java.nio.ByteBuffer +import kotlin.math.round + + +@Service +class CompressJudgements { + + val brotliParameters: Encoder.Parameters = Encoder.Parameters() + .setQuality(11) + + init { + Brotli4jLoader.ensureAvailability() + } + + fun serialize(judgements: List): ByteArray { + val buffer = ByteBuffer.allocate(judgements.size * (2 + 4 + 4 + 1 + 2 + 2 + 2)) + var lastTime = 0.0 + + judgements.forEach { judgement -> + val deltaTime = (judgement.time - lastTime).toInt() + buffer.putShort(deltaTime.toShort()) + + buffer.putInt((round(judgement.x * 1000)).toInt()) + buffer.putInt((round(judgement.y * 1000)).toInt()) + + buffer.put(judgement.type.ordinal.toByte()) + buffer.putShort((judgement.distance_center * 100).toInt().toShort()) + buffer.putShort((judgement.distance_edge * 100).toInt().toShort()) + buffer.putShort(judgement.error.toInt().toShort()) + + lastTime = judgement.time + } + return Encoder.compress(buffer.array(), brotliParameters) + } + + fun deserialize(compressedData: ByteArray): List { + val data = Decoder.decompress(compressedData).decompressedData + val buffer = ByteBuffer.wrap(data) + val judgements = mutableListOf() + var lastTime = 0.0 + + while (buffer.hasRemaining()) { + val deltaTime = buffer.short.toInt() + lastTime += deltaTime + + val deltaX = buffer.getInt() + val deltaY = buffer.getInt() + + val typeOrdinal = buffer.get().toInt() + val type = CircleguardService.JudgementType.entries[typeOrdinal] + + val distanceCenter = buffer.short.toInt() / 100.0 + val distanceEdge = buffer.short.toInt() / 100.0 + val error = buffer.short.toInt() + + judgements.add( + CircleguardService.ScoreJudgement( + time = lastTime, + x = deltaX / 1000.0, + y = deltaY / 1000.0, + type = type, + distance_center = distanceCenter, + distance_edge = distanceEdge, + error = error.toDouble() + )) + } + + return judgements + } + +} \ No newline at end of file diff --git a/nise-backend/src/main/resources/application-postgres.properties b/nise-backend/src/main/resources/application-postgres.properties index 64970cf..cfe8ac2 100644 --- a/nise-backend/src/main/resources/application-postgres.properties +++ b/nise-backend/src/main/resources/application-postgres.properties @@ -4,7 +4,7 @@ spring.datasource.password=${POSTGRES_PASS:postgres} spring.datasource.driver-class-name=org.postgresql.Driver spring.datasource.name=HikariPool-PostgreSQL -spring.flyway.enabled=true +spring.flyway.enabled=${FLYWAY_ENABLED:true} spring.flyway.schemas=public # Batching diff --git a/nise-backend/src/main/resources/db/migration/V0.0.1.024__alter_scores.sql b/nise-backend/src/main/resources/db/migration/V0.0.1.024__alter_scores.sql new file mode 100644 index 0000000..79b8a55 --- /dev/null +++ b/nise-backend/src/main/resources/db/migration/V0.0.1.024__alter_scores.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.scores + ADD COLUMN judgements bytea null; \ No newline at end of file diff --git a/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt new file mode 100644 index 0000000..012bcf4 --- /dev/null +++ b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt @@ -0,0 +1,81 @@ +package com.nisemoe.nise.scheduler + +import com.nisemoe.generated.tables.records.ScoresRecord +import com.nisemoe.generated.tables.references.SCORES +import com.nisemoe.nise.database.UserService +import com.nisemoe.nise.integrations.CircleguardService +import com.nisemoe.nise.osu.OsuApi +import com.nisemoe.nise.osu.TokenService +import com.nisemoe.nise.service.AuthService +import com.nisemoe.nise.service.CacheService +import com.nisemoe.nise.service.CompressJudgements +import kotlinx.serialization.json.Json +import org.jooq.DSLContext +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.annotation.Import +import org.springframework.test.context.ActiveProfiles + +@SpringBootTest +@ActiveProfiles("postgres") +@MockBean(GlobalCache::class, UserService::class) +@Disabled +class JudgementCompressionTest { + + private val logger = LoggerFactory.getLogger(javaClass) + private val circleguardService = CircleguardService() + private val compressJudgements = CompressJudgements() + + @Autowired + private lateinit var dslContext: DSLContext + + @Test + fun compressionIntegrityTest() { + + val scores = dslContext.select(SCORES.REPLAY, SCORES.BEATMAP_ID, SCORES.MODS) + .from(SCORES) + .where(SCORES.REPLAY.isNotNull) + .limit(100) + .fetchInto(ScoresRecord::class.java) + + for(score in scores) { + val result = score.replay?.let { + this.circleguardService.processReplay( + replayData = it.decodeToString(), + beatmapId = score.beatmapId!!, + mods = score.mods!! + ).get() + } + + if(result == null) { + this.logger.warn("Failed to process replay for beatmap {} with mods {}", score.beatmapId, score.mods) + continue + } + + val compressedData = compressJudgements.serialize(result.judgements) + + this.logger.info("JSON size: {} bytes", String.format("%,d", Json.encodeToString(CircleguardService.ReplayResponse.serializer(), result).length)) + this.logger.info("Compressed (Brotli) size: {} bytes", String.format("%,d", compressedData.size)) + + val deserializedData = compressJudgements.deserialize(compressedData) + + for (entry in result.judgements) { + assertEquals(entry.time, deserializedData[result.judgements.indexOf(entry)].time) + assertEquals(entry.x, deserializedData[result.judgements.indexOf(entry)].x, 0.01) + assertEquals(entry.y, deserializedData[result.judgements.indexOf(entry)].y, 0.01) + assertEquals(entry.type, deserializedData[result.judgements.indexOf(entry)].type) + assertEquals(entry.distance_center, deserializedData[result.judgements.indexOf(entry)].distance_center, 0.1) + assertEquals(entry.distance_edge, deserializedData[result.judgements.indexOf(entry)].distance_edge, 0.1) + assertEquals(entry.error, deserializedData[result.judgements.indexOf(entry)].error) + } + } + + } + +} \ No newline at end of file