Stedos/nise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Open sourced mari module

Browse Source
This commit is contained in:
nise.moe 2024-03-06 23:46:21 +01:00
parent ca9a43c06c
commit 402c89b20d
17 changed files with 389 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
mari/pom.xml Normal file
View File

@ -0,0 +1,112 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.nisemoe</groupId>
<artifactId>mari</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<java.version>21</java.version>
<kotlin.version>1.9.22</kotlin.version>
</properties>
<build>
<sourceDirectory>src/main/kotlin</sourceDirectory>
<testSourceDirectory>src/test/kotlin</testSourceDirectory>
<plugins>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
<version>${kotlin.version}</version>
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>test-compile</id>
<phase>test-compile</phase>
<goals>
<goal>test-compile</goal>
</goals>
</execution>
</executions>
<configuration>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
</configuration>
<dependencies>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-serialization</artifactId>
<version>${kotlin.version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<configuration>
<mainClass>MainKt</mainClass>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<!-- For Brotli compression of Judgement data -->
<dependency>
<groupId>com.aayushatharva.brotli4j</groupId>
<artifactId>brotli4j</artifactId>
<version>1.16.0</version>
</dependency>
<!-- For LZMA decompression of replay data -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.25.0</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>1.9</version>
</dependency>
<!-- For JSON serialization -->
<dependency>
<groupId>org.jetbrains.kotlinx</groupId>
<artifactId>kotlinx-serialization-json</artifactId>
<version>1.6.3</version>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-test-junit5</artifactId>
<version>${kotlin.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
</dependency>
</dependencies>
</project>

29
mari/readme.md Normal file
View File

@ -0,0 +1,29 @@
# mari
>osu! utility lib in kotlin to manipulate replays and judgement data in a safe and performant way.
This module allows [nise.moe](https://nise.moe) to juggle a ton of replays and data around.
# Usage
### Compress / decompress judgement data
The `Judgement` data class ought to represent the way a player has played a specific beatmap. It contains the hit error, distance, etc for each hit object. The structure is based off the Circleguard `Investigations.judgements` return type.
You can use `CompressJudgements.compress` and `CompressJudgements.decompress` to losslessly store and retrieve judgement data. According to my estimates, the compressed data is about 33% the size of the original data.
### Decode a replay
>This method is fundamentally written with the assumption that it'll be used on user-provided replays. It is thus designed to be safe and to not crash on invalid replays.
`OsuReplay` allows you to safely decode an `.osr` file. Once you've parsed the file, you can instantiate a new class:
```kotlin
val replay = OsuReplay(replayFile.bytes)
```
If everything goes well, you can then start reading the replay data.
```kotlin
println(replay.playerName) // mrekk
```

111
mari/src/main/kotlin/org/nisemoe/mari/judgements/CompressJudgements.kt Normal file
View File

@ -0,0 +1,111 @@
package org.nisemoe.mari.judgements
import com.aayushatharva.brotli4j.Brotli4jLoader
import com.aayushatharva.brotli4j.decoder.Decoder
import com.aayushatharva.brotli4j.encoder.Encoder
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
import kotlin.math.round
/**
* Compresses and decompresses score judgements with lossless accuracy.
*/
class CompressJudgements {
companion object {
init {
Brotli4jLoader.ensureAvailability()
}
private val brotliParameters: Encoder.Parameters = Encoder.Parameters()
.setQuality(11)
/**
* Writes a variable-length quantity to the buffer.
* See: https://en.wikipedia.org/wiki/Variable-length_quantity
*/
private fun ByteBuffer.putVLQ(value: Int) {
var currentValue = value
do {
var temp = (currentValue and 0x7F)
currentValue = currentValue ushr 7
if (currentValue != 0) {
temp = temp or 0x80
}
this.put(temp.toByte())
} while (currentValue != 0)
}
/**
* Reads a variable-length quantity from the buffer.
* See: https://en.wikipedia.org/wiki/Variable-length_quantity
*/
private fun ByteBuffer.getVLQ(): Int {
var result = 0
var shift = 0
var b: Byte
do {
b = this.get()
result = result or ((b.toInt() and 0x7F) shl shift)
shift += 7
} while (b.toInt() and 0x80 != 0)
return result
}
fun compress(judgements: List<Judgement>): ByteArray {
val byteStream = ByteArrayOutputStream()
var lastTimestamp = 0.0
judgements.forEach { judgement ->
byteStream.use { stream ->
/**
* We allocate an arbitrary amount of buffer which *hopefully* is enough.
*/
ByteBuffer.allocate(4096).let { buffer ->
buffer.putVLQ((judgement.time - lastTimestamp).toInt())
buffer.putVLQ(round(judgement.x * 100).toInt())
buffer.putVLQ(round(judgement.y * 100).toInt())
buffer.put(judgement.type.ordinal.toByte())
buffer.putVLQ((judgement.distanceToCenter * 100).toInt())
buffer.putVLQ((judgement.distanceToEdge * 100).toInt())
buffer.putVLQ(judgement.error.toInt())
lastTimestamp = judgement.time
stream.write(buffer.array(), 0, buffer.position())
}
}
}
return Encoder.compress(byteStream.toByteArray(), brotliParameters)
}
fun decompress(compressedData: ByteArray): List<Judgement> {
val data = Decoder.decompress(compressedData).decompressedData ?: return emptyList()
val buffer = ByteBuffer.wrap(data)
val judgements = mutableListOf<Judgement>()
var lastTime = 0.0
while (buffer.hasRemaining()) {
val deltaTime = buffer.getVLQ()
lastTime += deltaTime
judgements.add(
Judgement(
time = lastTime,
x = buffer.getVLQ() / 100.0,
y = buffer.getVLQ() / 100.0,
type = Judgement.Type.entries[buffer.get().toInt()],
distanceToCenter = buffer.getVLQ() / 100.0,
distanceToEdge = buffer.getVLQ() / 100.0,
error = buffer.getVLQ().toDouble()
))
}
return judgements
}
}
}

45
mari/src/main/kotlin/org/nisemoe/mari/judgements/JudgementModel.kt Normal file
View File

@ -0,0 +1,45 @@
package org.nisemoe.mari.judgements
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
/**
* Represents a judgement on a hit object.
* The structure *might* seem arbitrary but it's more or less what Circleguard provides.
*/
@Serializable
data class Judgement(
val time: Double,
val x: Double,
val y: Double,
val type: Type,
@SerialName("distance_center")
val distanceToCenter: Double,
@SerialName("distance_edge")
val distanceToEdge: Double,
val error: Double
) {
@Serializable
enum class Type {
@SerialName("Hit300")
THREE_HUNDRED,
@SerialName("Hit100")
ONE_HUNDRED,
@SerialName("Hit50")
FIFTY,
@SerialName("Miss")
MISS
}
}

36
nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuReplay.kt → mari/src/main/kotlin/org/nisemoe/mari/replays/OsuReplay.kt
View File

@ -1,4 +1,4 @@
package com.nisemoe.nise.osu
package org.nisemoe.mari.replays
import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream
import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream
@ -9,14 +9,21 @@ import java.nio.ByteOrder
import java.nio.charset.StandardCharsets
import java.util.*
/**
* Decodes an osu! replay file from a byte array.
*/
class OsuReplay(fileContent: ByteArray) {
companion object {
// ~4mb
/**
* We restrict the maximum replay file size to roughly 4MB.
*/
private val EXPECTED_FILE_SIZE = 0 .. 4194304
// ~512kb
/**
* We restrict the maximum string length to roughly 500KB.
*/
private const val MAX_STRING_LENGTH = 512000
private val EXPECTED_STRING_LENGTH = 0 .. MAX_STRING_LENGTH
@ -64,9 +71,6 @@ class OsuReplay(fileContent: ByteArray) {
private fun decode() {
try {
gameMode = dis.readByte().toInt()
if(gameMode != 0) {
throw SecurityException("Invalid game mode")
}
gameVersion = readIntLittleEndian()
beatmapHash = dis.readCompressedReplayData()
@ -109,22 +113,18 @@ class OsuReplay(fileContent: ByteArray) {
}
private fun DataInputStream.readCompressedReplayData(length: Int): String {
// Read the compressed data
val compressedData = ByteArray(length)
readFully(compressedData)
// Decompress the data
val decompressedStream = LZMACompressorInputStream(compressedData.inputStream())
val decompressedData = decompressedStream.readBytes()
decompressedStream.close()
val compressedOutputStream = ByteArrayOutputStream().use { outputStream ->
LZMACompressorInputStream(compressedData.inputStream()).use { decompressedStream ->
LZMACompressorOutputStream(outputStream).use { lzmaCompressorOutputStream ->
decompressedStream.copyTo(lzmaCompressorOutputStream)
}
}
outputStream
}
// Compress the decompressed data
val compressedOutputStream = ByteArrayOutputStream()
val lzmaCompressorOutputStream = LZMACompressorOutputStream(compressedOutputStream)
lzmaCompressorOutputStream.write(decompressedData)
lzmaCompressorOutputStream.close()
// Now encode the re-compressed data to Base64
return Base64.getEncoder().encodeToString(compressedOutputStream.toByteArray())
}

30
mari/src/test/kotlin/org/nisemoe/mari/judgements/CompressJudgementsTest.kt Normal file
View File

@ -0,0 +1,30 @@
package org.nisemoe.mari.judgements
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
class CompressJudgementsTest {
@Test
fun testCompress() {
val judgements = listOf(
Judgement(time = 1.0, x = 1.0, y = 1.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 1.0, distanceToEdge = 1.0, error = 1.0),
Judgement(time = 2.0, x = 2.0, y = 2.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 2.0, distanceToEdge = 2.0, error = 2.0)
)
val compressedData = CompressJudgements.compress(judgements)
assertTrue(compressedData.isNotEmpty())
}
@Test
fun testCompressAndDecompress() {
val originalJudgements = listOf(
Judgement(time = 1.123456789123456, x = 1.0, y = 1.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 1.0, distanceToEdge = 1.0, error = 1.0),
Judgement(time = 2.123456789123456, x = 2.0, y = 2.0, type = Judgement.Type.THREE_HUNDRED, distanceToCenter = 2.0, distanceToEdge = 2.0, error = 2.0)
)
val compressedData = CompressJudgements.compress(originalJudgements)
val decompressedJudgements = CompressJudgements.decompress(compressedData)
assertEquals(originalJudgements, decompressedJudgements)
}
}

5
nise-backend/pom.xml
View File

@ -64,6 +64,11 @@
<artifactId>konata</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.nisemoe</groupId>
<artifactId>mari</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>

12
nise-backend/src/main/kotlin/com/nisemoe/nise/Format.kt
View File

@ -1,7 +1,7 @@
package com.nisemoe.nise
import com.nisemoe.generated.enums.JudgementType
import com.nisemoe.nise.integrations.CircleguardService
import org.nisemoe.mari.judgements.Judgement
import java.time.LocalDateTime
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
@ -25,12 +25,12 @@ class Format {
return Date.from(localDateTime.atZone(ZoneOffset.UTC).toInstant())
}
fun fromJudgementType(circleGuardJudgementType: CircleguardService.JudgementType): JudgementType {
fun fromJudgementType(circleGuardJudgementType: Judgement.Type): JudgementType {
return when (circleGuardJudgementType) {
CircleguardService.JudgementType.THREE_HUNDRED -> JudgementType.`300`
CircleguardService.JudgementType.ONE_HUNDRED -> JudgementType.`100`
CircleguardService.JudgementType.FIFTY -> JudgementType.`50`
CircleguardService.JudgementType.MISS -> JudgementType.Miss
Judgement.Type.THREE_HUNDRED -> JudgementType.`300`
Judgement.Type.ONE_HUNDRED -> JudgementType.`100`
Judgement.Type.FIFTY -> JudgementType.`50`
Judgement.Type.MISS -> JudgementType.Miss
}
}

9
nise-backend/src/main/kotlin/com/nisemoe/nise/Models.kt
View File

@ -1,9 +1,8 @@
package com.nisemoe.nise
import com.nisemoe.nise.integrations.CircleguardService
import kotlinx.serialization.Serializable
import org.nisemoe.mari.judgements.Judgement
import java.time.OffsetDateTime
import java.util.UUID
data class UserQueueDetails(
val isProcessing: Boolean,
@ -100,7 +99,7 @@ data class ReplayViewerData(
val beatmap: String,
val replay: String,
val mods: Int,
val judgements: List<CircleguardService.ScoreJudgement>
val judgements: List<Judgement>
)
data class ReplayPairViewerData(
@ -108,8 +107,8 @@ data class ReplayPairViewerData(
val replay1: String,
val replay2: String,
val mods: Int,
val judgements1: List<CircleguardService.ScoreJudgement>,
val judgements2: List<CircleguardService.ScoreJudgement>
val judgements1: List<Judgement>,
val judgements2: List<Judgement>
)
data class ReplayData(

7
nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UploadReplayController.kt
View File

@ -10,10 +10,10 @@ import com.nisemoe.konata.compareSingleReplayWithSet
import com.nisemoe.nise.database.BeatmapService
import com.nisemoe.nise.integrations.CircleguardService
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.OsuReplay
import com.nisemoe.nise.scheduler.ImportScores
import com.nisemoe.nise.service.CompressJudgements
import org.jooq.DSLContext
import org.nisemoe.mari.judgements.CompressJudgements
import org.nisemoe.mari.replays.OsuReplay
import org.slf4j.LoggerFactory
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.PostMapping
@ -26,7 +26,6 @@ import java.time.OffsetDateTime
class UploadReplayController(
private val dslContext: DSLContext,
private val beatmapService: BeatmapService,
private val compressJudgements: CompressJudgements,
private val circleguardService: CircleguardService,
private val osuApi: OsuApi
) {
@ -175,7 +174,7 @@ class UploadReplayController(
.set(USER_SCORES.KEYPRESSES_STANDARD_DEVIATION_ADJUSTED, analysis.keypresses_standard_deviation_adjusted)
.set(USER_SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, analysis.sliderend_release_median_adjusted)
.set(USER_SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, analysis.sliderend_release_standard_deviation_adjusted)
.set(SCORES.JUDGEMENTS, compressJudgements.serialize(analysis.judgements))
.set(SCORES.JUDGEMENTS, CompressJudgements.compress(analysis.judgements))
.returning(USER_SCORES.ID)
.fetchOne()

29
nise-backend/src/main/kotlin/com/nisemoe/nise/database/ScoreService.kt
View File

@ -9,13 +9,14 @@ import com.nisemoe.nise.integrations.CircleguardService
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.service.AuthService
import com.nisemoe.nise.service.CompressJudgements
import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream
import org.jooq.Condition
import org.jooq.DSLContext
import org.jooq.Record
import org.jooq.impl.DSL
import org.jooq.impl.DSL.avg
import org.nisemoe.mari.judgements.CompressJudgements
import org.nisemoe.mari.judgements.Judgement
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
import java.time.LocalDateTime
@ -25,10 +26,8 @@ import kotlin.math.roundToInt
@Service
class ScoreService(
private val dslContext: DSLContext,
private val beatmapService: BeatmapService,
private val authService: AuthService,
private val osuApi: OsuApi,
private val compressJudgements: CompressJudgements
private val osuApi: OsuApi
) {
private val logger = LoggerFactory.getLogger(javaClass)
@ -502,16 +501,16 @@ class ScoreService(
replayData.comparable_adjusted_ur = otherScores.get("avg_adjusted_ur", Double::class.java)
}
fun mapLegacyJudgement(judgementType: JudgementType): CircleguardService.JudgementType {
fun mapLegacyJudgement(judgementType: JudgementType): Judgement.Type {
return when(judgementType) {
JudgementType.Miss -> CircleguardService.JudgementType.MISS
JudgementType.`300` -> CircleguardService.JudgementType.THREE_HUNDRED
JudgementType.`100` -> CircleguardService.JudgementType.ONE_HUNDRED
JudgementType.`50` -> CircleguardService.JudgementType.FIFTY
JudgementType.Miss -> Judgement.Type.MISS
JudgementType.`300` -> Judgement.Type.THREE_HUNDRED
JudgementType.`100` -> Judgement.Type.ONE_HUNDRED
JudgementType.`50` -> Judgement.Type.FIFTY
}
}
fun getJudgements(replayId: Long): List<CircleguardService.ScoreJudgement> {
fun getJudgements(replayId: Long): List<Judgement> {
val judgementsRecord = dslContext.select(SCORES.JUDGEMENTS)
.from(SCORES)
.where(SCORES.REPLAY_ID.eq(replayId))
@ -527,19 +526,19 @@ class ScoreService(
.where(SCORES_JUDGEMENTS.SCORE_ID.eq(scoreId))
.fetchInto(ScoresJudgementsRecord::class.java)
return judgementRecords.map {
CircleguardService.ScoreJudgement(
Judgement(
x = it.x!!,
y = it.y!!,
error = it.error!!,
distance_center = it.distanceCenter!!,
distance_edge = it.distanceEdge!!,
distanceToCenter = it.distanceCenter!!,
distanceToEdge = it.distanceEdge!!,
time = it.time!!,
type = mapLegacyJudgement(it.type!!)
)
}
}
return compressJudgements.deserialize(judgementsRecord.judgements!!)
return CompressJudgements.decompress(judgementsRecord.judgements!!)
}
fun getHitDistribution(scoreId: Int): Map<Int, DistributionEntry> {
@ -552,7 +551,7 @@ class ScoreService(
return this.getHitDistributionLegacy(scoreId)
}
val judgements = compressJudgements.deserialize(judgementsRecord.judgements!!)
val judgements = CompressJudgements.decompress(judgementsRecord.judgements!!)
val errorDistribution = mutableMapOf<Int, MutableMap<String, Int>>()
var totalHits = 0

7
nise-backend/src/main/kotlin/com/nisemoe/nise/database/UserScoreService.kt
View File

@ -6,8 +6,8 @@ import com.nisemoe.nise.Format
import com.nisemoe.nise.ReplayData
import com.nisemoe.nise.ReplayDataSimilarScore
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.service.CompressJudgements
import org.jooq.DSLContext
import org.nisemoe.mari.judgements.CompressJudgements
import org.springframework.stereotype.Service
import java.time.LocalDateTime
import java.util.*
@ -16,8 +16,7 @@ import kotlin.math.roundToInt
@Service
class UserScoreService(
private val dslContext: DSLContext,
private val scoreService: ScoreService,
private val compressJudgements: CompressJudgements
private val scoreService: ScoreService
) {
fun getReplayData(replayId: UUID): ReplayData? {
@ -165,7 +164,7 @@ class UserScoreService(
}
fun getHitDistribution(compressedJudgements: ByteArray): Map<Int, DistributionEntry> {
val judgements = compressJudgements.deserialize(compressedJudgements)
val judgements = CompressJudgements.decompress(compressedJudgements)
val errorDistribution = mutableMapOf<Int, MutableMap<String, Int>>()
var totalHits = 0

30
nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt
View File

@ -2,9 +2,9 @@ package com.nisemoe.nise.integrations
import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.scheduler.ImportScores
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import org.nisemoe.mari.judgements.Judgement
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Service
import java.net.URI
@ -32,32 +32,6 @@ class CircleguardService {
val mods: Int
)
@Serializable
data class ScoreJudgement(
val time: Double,
val x: Double,
val y: Double,
val type: JudgementType,
val distance_center: Double,
val distance_edge: Double,
val error: Double
)
@Serializable
enum class JudgementType {
@SerialName("Hit300")
THREE_HUNDRED,
@SerialName("Hit100")
ONE_HUNDRED,
@SerialName("Hit50")
FIFTY,
@SerialName("Miss")
MISS
}
@Serializable
data class ReplayResponse(
val ur: Double?,
@ -87,7 +61,7 @@ class CircleguardService {
val sliderend_release_standard_deviation: Double?,
val sliderend_release_standard_deviation_adjusted: Double?,
val judgements: List<ScoreJudgement>
val judgements: List<Judgement>
)
fun postProcessReplay(replayResponse: ReplayResponse, mods: Int = 0) {

7
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt
View File

@ -5,13 +5,13 @@ import com.nisemoe.generated.tables.references.BEATMAPS
import com.nisemoe.generated.tables.references.SCORES
import com.nisemoe.nise.integrations.CircleguardService
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.service.CompressJudgements
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.joinAll
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import org.jooq.DSLContext
import org.nisemoe.mari.judgements.CompressJudgements
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Value
import org.springframework.context.annotation.Profile
@ -24,8 +24,7 @@ import java.time.OffsetDateTime
class FixOldScores(
private val dslContext: DSLContext,
private val osuApi: OsuApi,
private val circleguardService: CircleguardService,
private val compressJudgements: CompressJudgements
private val circleguardService: CircleguardService
){
companion object {
@ -179,7 +178,7 @@ class FixOldScores(
.set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted)
.set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation)
.set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted)
.set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements))
.set(SCORES.JUDGEMENTS, CompressJudgements.compress(processedReplay.judgements))
.where(SCORES.REPLAY_ID.eq(score.replayId))
.returningResult(SCORES.ID)
.fetchOne()?.getValue(SCORES.ID)

7
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt
View File

@ -15,11 +15,11 @@ import com.nisemoe.nise.osu.Mod
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.OsuApiModels
import com.nisemoe.nise.service.CacheService
import com.nisemoe.nise.service.CompressJudgements
import com.nisemoe.nise.service.UpdateUserQueueService
import kotlinx.serialization.Serializable
import org.jooq.DSLContext
import org.jooq.Query
import org.nisemoe.mari.judgements.CompressJudgements
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.InitializingBean
import org.springframework.beans.factory.annotation.Value
@ -48,8 +48,7 @@ class ImportScores(
private val scoreService: ScoreService,
private val updateUserQueueService: UpdateUserQueueService,
private val circleguardService: CircleguardService,
private val messagingTemplate: SimpMessagingTemplate,
private val compressJudgements: CompressJudgements
private val messagingTemplate: SimpMessagingTemplate
) : InitializingBean {
private val userToUpdateBucket = mutableListOf<Long>()
@ -760,7 +759,7 @@ class ImportScores(
.set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted)
.set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation)
.set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted)
.set(SCORES.JUDGEMENTS, compressJudgements.serialize(processedReplay.judgements))
.set(SCORES.JUDGEMENTS, CompressJudgements.compress(processedReplay.judgements))
.where(SCORES.REPLAY_ID.eq(score.best_id))
.returningResult(SCORES.ID)
.fetchOne()?.getValue(SCORES.ID)

99
nise-backend/src/main/kotlin/com/nisemoe/nise/service/CompressJudgements.kt
View File

@ -1,99 +0,0 @@
package com.nisemoe.nise.service
import com.aayushatharva.brotli4j.Brotli4jLoader
import com.aayushatharva.brotli4j.decoder.Decoder
import com.aayushatharva.brotli4j.encoder.Encoder
import com.nisemoe.nise.integrations.CircleguardService
import org.springframework.stereotype.Service
import java.io.ByteArrayOutputStream
import java.nio.ByteBuffer
import kotlin.math.round
@Service
class CompressJudgements {
val brotliParameters: Encoder.Parameters = Encoder.Parameters()
.setQuality(11)
init {
Brotli4jLoader.ensureAvailability()
}
fun ByteBuffer.putVLQ(value: Int) {
var currentValue = value
do {
var temp = (currentValue and 0x7F)
currentValue = currentValue ushr 7
if (currentValue != 0) {
temp = temp or 0x80
}
this.put(temp.toByte())
} while (currentValue != 0)
}
fun ByteBuffer.getVLQ(): Int {
var result = 0
var shift = 0
var b: Byte
do {
b = this.get()
result = result or ((b.toInt() and 0x7F) shl shift)
shift += 7
} while (b.toInt() and 0x80 != 0)
return result
}
fun serialize(judgements: List<CircleguardService.ScoreJudgement>): ByteArray {
val byteStream = ByteArrayOutputStream()
var lastTimestamp = 0.0
judgements.forEach { judgement ->
byteStream.use { stream ->
/**
* We allocate an arbitrary amount of buffer which *hopefully* is enough.
*/
ByteBuffer.allocate(4096).let { buffer ->
buffer.putVLQ((judgement.time - lastTimestamp).toInt())
buffer.putVLQ(round(judgement.x * 100).toInt())
buffer.putVLQ(round(judgement.y * 100).toInt())
buffer.put(judgement.type.ordinal.toByte())
buffer.putVLQ((judgement.distance_center * 100).toInt())
buffer.putVLQ((judgement.distance_edge * 100).toInt())
buffer.putVLQ(judgement.error.toInt())
lastTimestamp = judgement.time
stream.write(buffer.array(), 0, buffer.position())
}
}
}
return Encoder.compress(byteStream.toByteArray(), brotliParameters)
}
fun deserialize(compressedData: ByteArray): List<CircleguardService.ScoreJudgement> {
val data = Decoder.decompress(compressedData).decompressedData ?: return emptyList()
val buffer = ByteBuffer.wrap(data)
val judgements = mutableListOf<CircleguardService.ScoreJudgement>()
var lastTime = 0.0
while (buffer.hasRemaining()) {
val deltaTime = buffer.getVLQ()
lastTime += deltaTime
judgements.add(
CircleguardService.ScoreJudgement(
time = lastTime,
x = buffer.getVLQ() / 100.0,
y = buffer.getVLQ() / 100.0,
type = CircleguardService.JudgementType.entries[buffer.get().toInt()],
distance_center = buffer.getVLQ() / 100.0,
distance_edge = buffer.getVLQ() / 100.0,
error = buffer.getVLQ().toDouble()
))
}
return judgements
}
}

8
nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/JudgementCompressionTest.kt
View File

@ -5,11 +5,7 @@ import com.nisemoe.generated.tables.references.BEATMAPS
import com.nisemoe.generated.tables.references.SCORES
import com.nisemoe.nise.database.UserService
import com.nisemoe.nise.integrations.CircleguardService
import com.nisemoe.nise.osu.OsuApi
import com.nisemoe.nise.osu.TokenService
import com.nisemoe.nise.service.AuthService
import com.nisemoe.nise.service.CacheService
import com.nisemoe.nise.service.CompressJudgements
import com.nisemoe.nise.osu.CompressJudgements
import kotlinx.serialization.json.Json
import org.jooq.DSLContext
import org.junit.jupiter.api.Assertions.*
@ -17,10 +13,8 @@ import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.context.annotation.Import
import org.springframework.test.context.ActiveProfiles
@SpringBootTest