diff --git a/mari/pom.xml b/mari/pom.xml new file mode 100644 index 0000000..c19f75d --- /dev/null +++ b/mari/pom.xml @@ -0,0 +1,112 @@ + + + 4.0.0 + + org.nisemoe + mari + 0.0.1-SNAPSHOT + + + 21 + 1.9.22 + + + + src/main/kotlin + src/test/kotlin + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + test-compile + test-compile + + test-compile + + + + + + kotlinx-serialization + + + + + org.jetbrains.kotlin + kotlin-maven-serialization + ${kotlin.version} + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + MainKt + + + + + + + + + com.aayushatharva.brotli4j + brotli4j + 1.16.0 + + + + + org.apache.commons + commons-compress + 1.25.0 + + + org.tukaani + xz + 1.9 + + + + + org.jetbrains.kotlinx + kotlinx-serialization-json + 1.6.3 + + + + org.jetbrains.kotlin + kotlin-test-junit5 + ${kotlin.version} + test + + + + org.junit.jupiter + junit-jupiter + 5.10.0 + test + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + + \ No newline at end of file diff --git a/mari/readme.md b/mari/readme.md new file mode 100644 index 0000000..374ef48 --- /dev/null +++ b/mari/readme.md @@ -0,0 +1,29 @@ +# mari + +>osu! utility lib in kotlin to manipulate replays and judgement data in a safe and performant way. + +This module allows [nise.moe](https://nise.moe) to juggle a ton of replays and data around. + +# Usage + +### Compress / decompress judgement data + +The `Judgement` data class ought to represent the way a player has played a specific beatmap. It contains the hit error, distance, etc for each hit object. The structure is based off the Circleguard `Investigations.judgements` return type. + +You can use `CompressJudgements.compress` and `CompressJudgements.decompress` to losslessly store and retrieve judgement data. According to my estimates, the compressed data is about 33% the size of the original data. + +### Decode a replay + +>This method is fundamentally written with the assumption that it'll be used on user-provided replays. It is thus designed to be safe and to not crash on invalid replays. + +`OsuReplay` allows you to safely decode an `.osr` file. Once you've parsed the file, you can instantiate a new class: + +```kotlin +val replay = OsuReplay(replayFile.bytes) +``` + +If everything goes well, you can then start reading the replay data. + +```kotlin +println(replay.playerName) // mrekk +``` diff --git a/mari/src/main/kotlin/org/nisemoe/mari/judgements/CompressJudgements.kt b/mari/src/main/kotlin/org/nisemoe/mari/judgements/CompressJudgements.kt new file mode 100644 index 0000000..19dd7b0 --- /dev/null +++ b/mari/src/main/kotlin/org/nisemoe/mari/judgements/CompressJudgements.kt @@ -0,0 +1,111 @@ +package org.nisemoe.mari.judgements + +import com.aayushatharva.brotli4j.Brotli4jLoader +import com.aayushatharva.brotli4j.decoder.Decoder +import com.aayushatharva.brotli4j.encoder.Encoder +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import kotlin.math.round + +/** + * Compresses and decompresses score judgements with lossless accuracy. + */ +class CompressJudgements { + + companion object { + + init { + Brotli4jLoader.ensureAvailability() + } + + private val brotliParameters: Encoder.Parameters = Encoder.Parameters() + .setQuality(11) + + /** + * Writes a variable-length quantity to the buffer. + * See: https://en.wikipedia.org/wiki/Variable-length_quantity + */ + private fun ByteBuffer.putVLQ(value: Int) { + var currentValue = value + do { + var temp = (currentValue and 0x7F) + currentValue = currentValue ushr 7 + if (currentValue != 0) { + temp = temp or 0x80 + } + this.put(temp.toByte()) + } while (currentValue != 0) + } + + /** + * Reads a variable-length quantity from the buffer. + * See: https://en.wikipedia.org/wiki/Variable-length_quantity + */ + private fun ByteBuffer.getVLQ(): Int { + var result = 0 + var shift = 0 + var b: Byte + do { + b = this.get() + result = result or ((b.toInt() and 0x7F) shl shift) + shift += 7 + } while (b.toInt() and 0x80 != 0) + return result + } + + fun compress(judgements: List): ByteArray { + val byteStream = ByteArrayOutputStream() + var lastTimestamp = 0.0 + + judgements.forEach { judgement -> + byteStream.use { stream -> + /** + * We allocate an arbitrary amount of buffer which *hopefully* is enough. + */ + ByteBuffer.allocate(4096).let { buffer -> + buffer.putVLQ((judgement.time - lastTimestamp).toInt()) + buffer.putVLQ(round(judgement.x * 100).toInt()) + buffer.putVLQ(round(judgement.y * 100).toInt()) + buffer.put(judgement.type.ordinal.toByte()) + buffer.putVLQ((judgement.distanceToCenter * 100).toInt()) + buffer.putVLQ((judgement.distanceToEdge * 100).toInt()) + buffer.putVLQ(judgement.error.toInt()) + + lastTimestamp = judgement.time + stream.write(buffer.array(), 0, buffer.position()) + } + } + } + + return Encoder.compress(byteStream.toByteArray(), brotliParameters) + } + + fun decompress(compressedData: ByteArray): List { + val data = Decoder.decompress(compressedData).decompressedData ?: return emptyList() + + val buffer = ByteBuffer.wrap(data) + val judgements = mutableListOf() + var lastTime = 0.0 + + while (buffer.hasRemaining()) { + val deltaTime = buffer.getVLQ() + lastTime += deltaTime + + judgements.add( + Judgement( + time = lastTime, + x = buffer.getVLQ() / 100.0, + y = buffer.getVLQ() / 100.0, + type = Judgement.Type.entries[buffer.get().toInt()], + distanceToCenter = buffer.getVLQ() / 100.0, + distanceToEdge = buffer.getVLQ() / 100.0, + error = buffer.getVLQ().toDouble() + )) + } + + return judgements + } + + } + +} \ No newline at end of file diff --git a/mari/src/main/kotlin/org/nisemoe/mari/judgements/JudgementModel.kt b/mari/src/main/kotlin/org/nisemoe/mari/judgements/JudgementModel.kt new file mode 100644 index 0000000..2b5a111 --- /dev/null +++ b/mari/src/main/kotlin/org/nisemoe/mari/judgements/JudgementModel.kt @@ -0,0 +1,45 @@ +package org.nisemoe.mari.judgements + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Represents a judgement on a hit object. + * The structure *might* seem arbitrary but it's more or less what Circleguard provides. + */ +@Serializable +data class Judgement( + val time: Double, + val x: Double, + val y: Double, + val type: Type, + + @SerialName("distance_center") + val distanceToCenter: Double, + + @SerialName("distance_edge") + val distanceToEdge: Double, + + + val error: Double +) { + + @Serializable + enum class Type { + + @SerialName("Hit300") + THREE_HUNDRED, + + @SerialName("Hit100") + ONE_HUNDRED, + + @SerialName("Hit50") + FIFTY, + + @SerialName("Miss") + MISS + + } + +} + diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuReplay.kt b/mari/src/main/kotlin/org/nisemoe/mari/replays/OsuReplay.kt similarity index 89% rename from nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuReplay.kt rename to mari/src/main/kotlin/org/nisemoe/mari/replays/OsuReplay.kt index 33c1830..cbf0fb9 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuReplay.kt +++ b/mari/src/main/kotlin/org/nisemoe/mari/replays/OsuReplay.kt @@ -1,4 +1,4 @@ -package com.nisemoe.nise.osu +package org.nisemoe.mari.replays import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream @@ -9,14 +9,21 @@ import java.nio.ByteOrder import java.nio.charset.StandardCharsets import java.util.* +/** + * Decodes an osu! replay file from a byte array. + */ class OsuReplay(fileContent: ByteArray) { companion object { - // ~4mb + /** + * We restrict the maximum replay file size to roughly 4MB. + */ private val EXPECTED_FILE_SIZE = 0 .. 4194304 - // ~512kb + /** + * We restrict the maximum string length to roughly 500KB. + */ private const val MAX_STRING_LENGTH = 512000 private val EXPECTED_STRING_LENGTH = 0 .. MAX_STRING_LENGTH @@ -64,9 +71,6 @@ class OsuReplay(fileContent: ByteArray) { private fun decode() { try { gameMode = dis.readByte().toInt() - if(gameMode != 0) { - throw SecurityException("Invalid game mode") - } gameVersion = readIntLittleEndian() beatmapHash = dis.readCompressedReplayData() @@ -109,22 +113,18 @@ class OsuReplay(fileContent: ByteArray) { } private fun DataInputStream.readCompressedReplayData(length: Int): String { - // Read the compressed data val compressedData = ByteArray(length) readFully(compressedData) - // Decompress the data - val decompressedStream = LZMACompressorInputStream(compressedData.inputStream()) - val decompressedData = decompressedStream.readBytes() - decompressedStream.close() + val compressedOutputStream = ByteArrayOutputStream().use { outputStream -> + LZMACompressorInputStream(compressedData.inputStream()).use { decompressedStream -> + LZMACompressorOutputStream(outputStream).use { lzmaCompressorOutputStream -> + decompressedStream.copyTo(lzmaCompressorOutputStream) + } + } + outputStream + } - // Compress the decompressed data - val compressedOutputStream = ByteArrayOutputStream() - val lzmaCompressorOutputStream = LZMACompressorOutputStream(compressedOutputStream) - lzmaCompressorOutputStream.write(decompressedData) - lzmaCompressorOutputStream.close() - - // Now encode the re-compressed data to Base64 return Base64.getEncoder().encodeToString(compressedOutputStream.toByteArray()) } diff --git a/mari/src/test/kotlin/org/nisemoe/mari/judgements/CompressJudgementsTest.kt b/mari/src/test/kotlin/org/nisemoe/mari/judgements/CompressJudgementsTest.kt new file mode 100644 index 0000000..9d5f761 --- /dev/null +++ b/mari/src/test/kotlin/org/nisemoe/mari/judgements/CompressJudgementsTest.kt @@ -0,0 +1,30 @@ +package org.nisemoe.mari.judgements + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class CompressJudgementsTest { + + @Test + fun testCompress() { + val judgements = listOf( + Judgement(time = 1.0, x = 1.0, y = 1.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 1.0, distanceToEdge = 1.0, error = 1.0), + Judgement(time = 2.0, x = 2.0, y = 2.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 2.0, distanceToEdge = 2.0, error = 2.0) + ) + val compressedData = CompressJudgements.compress(judgements) + assertTrue(compressedData.isNotEmpty()) + } + + @Test + fun testCompressAndDecompress() { + val originalJudgements = listOf( + Judgement(time = 1.123456789123456, x = 1.0, y = 1.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 1.0, distanceToEdge = 1.0, error = 1.0), + Judgement(time = 2.123456789123456, x = 2.0, y = 2.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 2.0, distanceToEdge = 2.0, error = 2.0) + ) + val compressedData = CompressJudgements.compress(originalJudgements) + val decompressedJudgements = CompressJudgements.decompress(compressedData) + assertEquals(originalJudgements, decompressedJudgements) + } + +} \ No newline at end of file diff --git a/nise-backend/pom.xml b/nise-backend/pom.xml index 8d165bd..072aeb5 100644 --- a/nise-backend/pom.xml +++ b/nise-backend/pom.xml @@ -64,6 +64,11 @@ konata 0.0.1-SNAPSHOT + + org.nisemoe + mari + 0.0.1-SNAPSHOT + com.fasterxml.jackson.dataformat jackson-dataformat-xml diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/Format.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/Format.kt index 64ca405..6761b7f 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/Format.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/Format.kt @@ -1,7 +1,7 @@ package com.nisemoe.nise import com.nisemoe.generated.enums.JudgementType -import com.nisemoe.nise.integrations.CircleguardService +import org.nisemoe.mari.judgements.Judgement import java.time.LocalDateTime import java.time.ZoneOffset import java.time.format.DateTimeFormatter @@ -25,12 +25,12 @@ class Format { return Date.from(localDateTime.atZone(ZoneOffset.UTC).toInstant()) } - fun fromJudgementType(circleGuardJudgementType: CircleguardService.JudgementType): JudgementType { + fun fromJudgementType(circleGuardJudgementType: Judgement.Type): JudgementType { return when (circleGuardJudgementType) { - CircleguardService.JudgementType.THREE_HUNDRED -> JudgementType.`300` - CircleguardService.JudgementType.ONE_HUNDRED -> JudgementType.`100` - CircleguardService.JudgementType.FIFTY -> JudgementType.`50` - CircleguardService.JudgementType.MISS -> JudgementType.Miss + Judgement.Type.THREE_HUNDRED -> JudgementType.`300` + Judgement.Type.ONE_HUNDRED -> JudgementType.`100` + Judgement.Type.FIFTY -> JudgementType.`50` + Judgement.Type.MISS -> JudgementType.Miss } } diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt index a3a4f71..c2bd258 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt @@ -1,9 +1,8 @@ package com.nisemoe.nise -import com.nisemoe.nise.integrations.CircleguardService import kotlinx.serialization.Serializable +import org.nisemoe.mari.judgements.Judgement import java.time.OffsetDateTime -import java.util.UUID data class UserQueueDetails( val isProcessing: Boolean, @@ -100,7 +99,7 @@ data class ReplayViewerData( val beatmap: String, val replay: String, val mods: Int, - val judgements: List + val judgements: List ) data class ReplayPairViewerData( @@ -108,8 +107,8 @@ data class ReplayPairViewerData( val replay1: String, val replay2: String, val mods: Int, - val judgements1: List, - val judgements2: List + val judgements1: List, + val judgements2: List ) data class ReplayData( diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UploadReplayController.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UploadReplayController.kt index eb62822..0354382 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UploadReplayController.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UploadReplayController.kt @@ -10,10 +10,10 @@ import com.nisemoe.konata.compareSingleReplayWithSet import com.nisemoe.nise.database.BeatmapService import com.nisemoe.nise.integrations.CircleguardService import com.nisemoe.nise.osu.OsuApi -import com.nisemoe.nise.osu.OsuReplay import com.nisemoe.nise.scheduler.ImportScores -import com.nisemoe.nise.service.CompressJudgements import org.jooq.DSLContext +import org.nisemoe.mari.judgements.CompressJudgements +import org.nisemoe.mari.replays.OsuReplay import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.PostMapping @@ -26,7 +26,6 @@ import java.time.OffsetDateTime class UploadReplayController( private val dslContext: DSLContext, private val beatmapService: BeatmapService, - private val compressJudgements: CompressJudgements, private val circleguardService: CircleguardService, private val osuApi: OsuApi ) { @@ -175,7 +174,7 @@ class UploadReplayController( .set(USER_SCORES.KEYPRESSES_STANDARD_DEVIATION_ADJUSTED, analysis.keypresses_standard_deviation_adjusted) .set(USER_SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, analysis.sliderend_release_median_adjusted) .set(USER_SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, analysis.sliderend_release_standard_deviation_adjusted) - .set(SCORES.JUDGEMENTS, compressJudgements.serialize(analysis.judgements)) + .set(SCORES.JUDGEMENTS, CompressJudgements.compress(analysis.judgements)) .returning(USER_SCORES.ID) .fetchOne() diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt index 09b22d8..3b85cf8 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt @@ -9,13 +9,14 @@ import com.nisemoe.nise.integrations.CircleguardService import com.nisemoe.nise.osu.Mod import com.nisemoe.nise.osu.OsuApi import com.nisemoe.nise.service.AuthService -import com.nisemoe.nise.service.CompressJudgements import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Record import org.jooq.impl.DSL import org.jooq.impl.DSL.avg +import org.nisemoe.mari.judgements.CompressJudgements +import org.nisemoe.mari.judgements.Judgement import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.time.LocalDateTime @@ -25,10 +26,8 @@ import kotlin.math.roundToInt @Service class ScoreService( private val dslContext: DSLContext, - private val beatmapService: BeatmapService, private val authService: AuthService, - private val osuApi: OsuApi, - private val compressJudgements: CompressJudgements + private val osuApi: OsuApi ) { private val logger = LoggerFactory.getLogger(javaClass) @@ -502,16 +501,16 @@ class ScoreService( replayData.comparable_adjusted_ur = otherScores.get("avg_adjusted_ur", Double::class.java) } - fun mapLegacyJudgement(judgementType: JudgementType): CircleguardService.JudgementType { + fun mapLegacyJudgement(judgementType: JudgementType): Judgement.Type { return when(judgementType) { - JudgementType.Miss -> CircleguardService.JudgementType.MISS - JudgementType.`300` -> CircleguardService.JudgementType.THREE_HUNDRED - JudgementType.`100` -> CircleguardService.JudgementType.ONE_HUNDRED - JudgementType.`50` -> CircleguardService.JudgementType.FIFTY + JudgementType.Miss -> Judgement.Type.MISS + JudgementType.`300` -> Judgement.Type.THREE_HUNDRED + JudgementType.`100` -> Judgement.Type.ONE_HUNDRED + JudgementType.`50` -> Judgement.Type.FIFTY } } - fun getJudgements(replayId: Long): List { + fun getJudgements(replayId: Long): List { val judgementsRecord = dslContext.select(SCORES.JUDGEMENTS) .from(SCORES) .where(SCORES.REPLAY_ID.eq(replayId)) @@ -527,19 +526,19 @@ class ScoreService( .where(SCORES_JUDGEMENTS.SCORE_ID.eq(scoreId)) .fetchInto(ScoresJudgementsRecord::class.java) return judgementRecords.map { - CircleguardService.ScoreJudgement( + Judgement( x = it.x!!, y = it.y!!, error = it.error!!, - distance_center = it.distanceCenter!!, - distance_edge = it.distanceEdge!!, + distanceToCenter = it.distanceCenter!!, + distanceToEdge = it.distanceEdge!!, time = it.time!!, type = mapLegacyJudgement(it.type!!) ) } } - return compressJudgements.deserialize(judgementsRecord.judgements!!) + return CompressJudgements.decompress(judgementsRecord.judgements!!) } fun getHitDistribution(scoreId: Int): Map { @@ -552,7 +551,7 @@ class ScoreService( return this.getHitDistributionLegacy(scoreId) } - val judgements = compressJudgements.deserialize(judgementsRecord.judgements!!) + val judgements = CompressJudgements.decompress(judgementsRecord.judgements!!) val errorDistribution = mutableMapOf>() var totalHits = 0 diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt index 42d9b87..23ec651 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt @@ -6,8 +6,8 @@ import com.nisemoe.nise.Format import com.nisemoe.nise.ReplayData import com.nisemoe.nise.ReplayDataSimilarScore import com.nisemoe.nise.osu.Mod -import com.nisemoe.nise.service.CompressJudgements import org.jooq.DSLContext +import org.nisemoe.mari.judgements.CompressJudgements import org.springframework.stereotype.Service import java.time.LocalDateTime import java.util.* @@ -16,8 +16,7 @@ import kotlin.math.roundToInt @Service class UserScoreService( private val dslContext: DSLContext, - private val scoreService: ScoreService, - private val compressJudgements: CompressJudgements + private val scoreService: ScoreService ) { fun getReplayData(replayId: UUID): ReplayData? { @@ -165,7 +164,7 @@ class UserScoreService( } fun getHitDistribution(compressedJudgements: ByteArray): Map { - val judgements = compressJudgements.deserialize(compressedJudgements) + val judgements = CompressJudgements.decompress(compressedJudgements) val errorDistribution = mutableMapOf>() var totalHits = 0 diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt index 4360ed5..d756096 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt @@ -2,9 +2,9 @@ package com.nisemoe.nise.integrations import com.nisemoe.nise.osu.Mod import com.nisemoe.nise.scheduler.ImportScores -import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json +import org.nisemoe.mari.judgements.Judgement import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service import java.net.URI @@ -32,32 +32,6 @@ class CircleguardService { val mods: Int ) - @Serializable - data class ScoreJudgement( - val time: Double, - val x: Double, - val y: Double, - val type: JudgementType, - val distance_center: Double, - val distance_edge: Double, - val error: Double - ) - - @Serializable - enum class JudgementType { - @SerialName("Hit300") - THREE_HUNDRED, - - @SerialName("Hit100") - ONE_HUNDRED, - - @SerialName("Hit50") - FIFTY, - - @SerialName("Miss") - MISS - } - @Serializable data class ReplayResponse( val ur: Double?, @@ -87,7 +61,7 @@ class CircleguardService { val sliderend_release_standard_deviation: Double?, val sliderend_release_standard_deviation_adjusted: Double?, - val judgements: List + val judgements: List ) fun postProcessReplay(replayResponse: ReplayResponse, mods: Int = 0) { diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt index 00ab659..f63e382 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt @@ -5,13 +5,13 @@ import com.nisemoe.generated.tables.references.BEATMAPS import com.nisemoe.generated.tables.references.SCORES import com.nisemoe.nise.integrations.CircleguardService import com.nisemoe.nise.osu.OsuApi -import com.nisemoe.nise.service.CompressJudgements import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import org.jooq.DSLContext +import org.nisemoe.mari.judgements.CompressJudgements import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Profile @@ -24,8 +24,7 @@ import java.time.OffsetDateTime class FixOldScores( private val dslContext: DSLContext, private val osuApi: OsuApi, - private val circleguardService: CircleguardService, - private val compressJudgements: CompressJudgements + private val circleguardService: CircleguardService ){ companion object { @@ -179,7 +178,7 @@ class FixOldScores( .set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted) - .set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements)) + .set(SCORES.JUDGEMENTS, CompressJudgements.compress(processedReplay.judgements)) .where(SCORES.REPLAY_ID.eq(score.replayId)) .returningResult(SCORES.ID) .fetchOne()?.getValue(SCORES.ID) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index c3b8908..d7a5b16 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -15,11 +15,11 @@ import com.nisemoe.nise.osu.Mod import com.nisemoe.nise.osu.OsuApi import com.nisemoe.nise.osu.OsuApiModels import com.nisemoe.nise.service.CacheService -import com.nisemoe.nise.service.CompressJudgements import com.nisemoe.nise.service.UpdateUserQueueService import kotlinx.serialization.Serializable import org.jooq.DSLContext import org.jooq.Query +import org.nisemoe.mari.judgements.CompressJudgements import org.slf4j.LoggerFactory import org.springframework.beans.factory.InitializingBean import org.springframework.beans.factory.annotation.Value @@ -48,8 +48,7 @@ class ImportScores( private val scoreService: ScoreService, private val updateUserQueueService: UpdateUserQueueService, private val circleguardService: CircleguardService, - private val messagingTemplate: SimpMessagingTemplate, - private val compressJudgements: CompressJudgements + private val messagingTemplate: SimpMessagingTemplate ) : InitializingBean { private val userToUpdateBucket = mutableListOf() @@ -760,7 +759,7 @@ class ImportScores( .set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation) .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted) - .set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements)) + .set(SCORES.JUDGEMENTS, CompressJudgements.compress(processedReplay.judgements)) .where(SCORES.REPLAY_ID.eq(score.best_id)) .returningResult(SCORES.ID) .fetchOne()?.getValue(SCORES.ID) diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt deleted file mode 100644 index 24a1597..0000000 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt +++ /dev/null @@ -1,99 +0,0 @@ -package com.nisemoe.nise.service - -import com.aayushatharva.brotli4j.Brotli4jLoader -import com.aayushatharva.brotli4j.decoder.Decoder -import com.aayushatharva.brotli4j.encoder.Encoder -import com.nisemoe.nise.integrations.CircleguardService -import org.springframework.stereotype.Service -import java.io.ByteArrayOutputStream -import java.nio.ByteBuffer -import kotlin.math.round - -@Service -class CompressJudgements { - - val brotliParameters: Encoder.Parameters = Encoder.Parameters() - .setQuality(11) - - init { - Brotli4jLoader.ensureAvailability() - } - - fun ByteBuffer.putVLQ(value: Int) { - var currentValue = value - do { - var temp = (currentValue and 0x7F) - currentValue = currentValue ushr 7 - if (currentValue != 0) { - temp = temp or 0x80 - } - this.put(temp.toByte()) - } while (currentValue != 0) - } - - fun ByteBuffer.getVLQ(): Int { - var result = 0 - var shift = 0 - var b: Byte - do { - b = this.get() - result = result or ((b.toInt() and 0x7F) shl shift) - shift += 7 - } while (b.toInt() and 0x80 != 0) - return result - } - - fun serialize(judgements: List): ByteArray { - val byteStream = ByteArrayOutputStream() - var lastTimestamp = 0.0 - - judgements.forEach { judgement -> - byteStream.use { stream -> - /** - * We allocate an arbitrary amount of buffer which *hopefully* is enough. - */ - ByteBuffer.allocate(4096).let { buffer -> - buffer.putVLQ((judgement.time - lastTimestamp).toInt()) - buffer.putVLQ(round(judgement.x * 100).toInt()) - buffer.putVLQ(round(judgement.y * 100).toInt()) - buffer.put(judgement.type.ordinal.toByte()) - buffer.putVLQ((judgement.distance_center * 100).toInt()) - buffer.putVLQ((judgement.distance_edge * 100).toInt()) - buffer.putVLQ(judgement.error.toInt()) - - lastTimestamp = judgement.time - stream.write(buffer.array(), 0, buffer.position()) - } - } - } - - return Encoder.compress(byteStream.toByteArray(), brotliParameters) - } - - fun deserialize(compressedData: ByteArray): List { - val data = Decoder.decompress(compressedData).decompressedData ?: return emptyList() - - val buffer = ByteBuffer.wrap(data) - val judgements = mutableListOf() - var lastTime = 0.0 - - while (buffer.hasRemaining()) { - val deltaTime = buffer.getVLQ() - lastTime += deltaTime - - judgements.add( - CircleguardService.ScoreJudgement( - time = lastTime, - x = buffer.getVLQ() / 100.0, - y = buffer.getVLQ() / 100.0, - type = CircleguardService.JudgementType.entries[buffer.get().toInt()], - distance_center = buffer.getVLQ() / 100.0, - distance_edge = buffer.getVLQ() / 100.0, - error = buffer.getVLQ().toDouble() - )) - } - - return judgements - } - -} \ No newline at end of file diff --git a/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt index 81293a1..83b0f0a 100644 --- a/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt +++ b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt @@ -5,11 +5,7 @@ import com.nisemoe.generated.tables.references.BEATMAPS import com.nisemoe.generated.tables.references.SCORES import com.nisemoe.nise.database.UserService import com.nisemoe.nise.integrations.CircleguardService -import com.nisemoe.nise.osu.OsuApi -import com.nisemoe.nise.osu.TokenService -import com.nisemoe.nise.service.AuthService -import com.nisemoe.nise.service.CacheService -import com.nisemoe.nise.service.CompressJudgements +import com.nisemoe.nise.osu.CompressJudgements import kotlinx.serialization.json.Json import org.jooq.DSLContext import org.junit.jupiter.api.Assertions.* @@ -17,10 +13,8 @@ import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration import org.springframework.boot.test.context.SpringBootTest import org.springframework.boot.test.mock.mockito.MockBean -import org.springframework.context.annotation.Import import org.springframework.test.context.ActiveProfiles @SpringBootTest