Added adjusted (filters out outliers) keypress and slider end times

nise-backend/src/main/kotlin/com/nisemoe/generated/tables/Scores.kt

@@ -275,6 +275,28 @@ open class Scores(
      */
     val SLIDEREND_RELEASE_STANDARD_DEVIATION: TableField<ScoresRecord, Double?> = createField(DSL.name("sliderend_release_standard_deviation"), SQLDataType.DOUBLE, this, "")
 
+    /**
+     * The column <code>public.scores.keypresses_median_adjusted</code>.
+     */
+    val KEYPRESSES_MEDIAN_ADJUSTED: TableField<ScoresRecord, Double?> = createField(DSL.name("keypresses_median_adjusted"), SQLDataType.DOUBLE, this, "")
+
+    /**
+     * The column
+     * <code>public.scores.keypresses_standard_deviation_adjusted</code>.
+     */
+    val KEYPRESSES_STANDARD_DEVIATION_ADJUSTED: TableField<ScoresRecord, Double?> = createField(DSL.name("keypresses_standard_deviation_adjusted"), SQLDataType.DOUBLE, this, "")
+
+    /**
+     * The column <code>public.scores.sliderend_release_median_adjusted</code>.
+     */
+    val SLIDEREND_RELEASE_MEDIAN_ADJUSTED: TableField<ScoresRecord, Double?> = createField(DSL.name("sliderend_release_median_adjusted"), SQLDataType.DOUBLE, this, "")
+
+    /**
+     * The column
+     * <code>public.scores.sliderend_release_standard_deviation_adjusted</code>.
+     */
+    val SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED: TableField<ScoresRecord, Double?> = createField(DSL.name("sliderend_release_standard_deviation_adjusted"), SQLDataType.DOUBLE, this, "")
+
     private constructor(alias: Name, aliased: Table<ScoresRecord>?): this(alias, null, null, aliased, null)
     private constructor(alias: Name, aliased: Table<ScoresRecord>?, parameters: Array<Field<*>?>?): this(alias, null, null, aliased, parameters)
 

nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/ScoresRecord.kt

@@ -185,6 +185,22 @@ open class ScoresRecord private constructor() : UpdatableRecordImpl<ScoresRecord
         set(value): Unit = set(40, value)
         get(): Double? = get(40) as Double?
 
+    open var keypressesMedianAdjusted: Double?
+        set(value): Unit = set(41, value)
+        get(): Double? = get(41) as Double?
+
+    open var keypressesStandardDeviationAdjusted: Double?
+        set(value): Unit = set(42, value)
+        get(): Double? = get(42) as Double?
+
+    open var sliderendReleaseMedianAdjusted: Double?
+        set(value): Unit = set(43, value)
+        get(): Double? = get(43) as Double?
+
+    open var sliderendReleaseStandardDeviationAdjusted: Double?
+        set(value): Unit = set(44, value)
+        get(): Double? = get(44) as Double?
+
     // -------------------------------------------------------------------------
     // Primary key information
     // -------------------------------------------------------------------------
@@ -194,7 +210,7 @@ open class ScoresRecord private constructor() : UpdatableRecordImpl<ScoresRecord
     /**
      * Create a detached, initialised ScoresRecord
      */
-    constructor(id: Int? = null, beatmapId: Int? = null, count_100: Int? = null, count_300: Int? = null, count_50: Int? = null, countMiss: Int? = null, date: LocalDateTime? = null, maxCombo: Int? = null, mods: Int? = null, perfect: Boolean? = null, pp: Double? = null, rank: String? = null, replayAvailable: Boolean? = null, replayId: Long? = null, score: Long? = null, userId: Long? = null, replay: ByteArray? = null, ur: Double? = null, frametime: Double? = null, edgeHits: Int? = null, snaps: Int? = null, isBanned: Boolean? = null, adjustedUr: Double? = null, meanError: Double? = null, errorVariance: Double? = null, errorStandardDeviation: Double? = null, minimumError: Double? = null, maximumError: Double? = null, errorRange: Double? = null, errorCoefficientOfVariation: Double? = null, errorKurtosis: Double? = null, errorSkewness: Double? = null, sentDiscordNotification: Boolean? = null, addedAt: OffsetDateTime? = null, version: Int? = null, keypressesTimes: Array<Double?>? = null, keypressesMedian: Double? = null, keypressesStandardDeviation: Double? = null, sliderendReleaseTimes: Array<Double?>? = null, sliderendReleaseMedian: Double? = null, sliderendReleaseStandardDeviation: Double? = null): this() {
+    constructor(id: Int? = null, beatmapId: Int? = null, count_100: Int? = null, count_300: Int? = null, count_50: Int? = null, countMiss: Int? = null, date: LocalDateTime? = null, maxCombo: Int? = null, mods: Int? = null, perfect: Boolean? = null, pp: Double? = null, rank: String? = null, replayAvailable: Boolean? = null, replayId: Long? = null, score: Long? = null, userId: Long? = null, replay: ByteArray? = null, ur: Double? = null, frametime: Double? = null, edgeHits: Int? = null, snaps: Int? = null, isBanned: Boolean? = null, adjustedUr: Double? = null, meanError: Double? = null, errorVariance: Double? = null, errorStandardDeviation: Double? = null, minimumError: Double? = null, maximumError: Double? = null, errorRange: Double? = null, errorCoefficientOfVariation: Double? = null, errorKurtosis: Double? = null, errorSkewness: Double? = null, sentDiscordNotification: Boolean? = null, addedAt: OffsetDateTime? = null, version: Int? = null, keypressesTimes: Array<Double?>? = null, keypressesMedian: Double? = null, keypressesStandardDeviation: Double? = null, sliderendReleaseTimes: Array<Double?>? = null, sliderendReleaseMedian: Double? = null, sliderendReleaseStandardDeviation: Double? = null, keypressesMedianAdjusted: Double? = null, keypressesStandardDeviationAdjusted: Double? = null, sliderendReleaseMedianAdjusted: Double? = null, sliderendReleaseStandardDeviationAdjusted: Double? = null): this() {
         this.id = id
         this.beatmapId = beatmapId
         this.count_100 = count_100
@@ -236,6 +252,10 @@ open class ScoresRecord private constructor() : UpdatableRecordImpl<ScoresRecord
         this.sliderendReleaseTimes = sliderendReleaseTimes
         this.sliderendReleaseMedian = sliderendReleaseMedian
         this.sliderendReleaseStandardDeviation = sliderendReleaseStandardDeviation
+        this.keypressesMedianAdjusted = keypressesMedianAdjusted
+        this.keypressesStandardDeviationAdjusted = keypressesStandardDeviationAdjusted
+        this.sliderendReleaseMedianAdjusted = sliderendReleaseMedianAdjusted
+        this.sliderendReleaseStandardDeviationAdjusted = sliderendReleaseStandardDeviationAdjusted
         resetChangedOnNotNull()
     }
 }

nise-backend/src/main/kotlin/com/nisemoe/nise/integrations/CircleguardService.kt

@@ -77,11 +77,15 @@ class CircleguardService {
 
         val keypresses_times: List<Double>?,
         val keypresses_median: Double?,
+        val keypresses_median_adjusted: Double?,
         val keypresses_standard_deviation: Double?,
+        val keypresses_standard_deviation_adjusted: Double?,
 
         val sliderend_release_times: List<Double>?,
         val sliderend_release_median: Double?,
+        val sliderend_release_median_adjusted: Double?,
         val sliderend_release_standard_deviation: Double?,
+        val sliderend_release_standard_deviation_adjusted: Double?,
 
         val judgements: List<ScoreJudgement>
     )

nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/FixOldScores.kt

@@ -31,7 +31,7 @@ class FixOldScores(
     @Value("\${OLD_SCORES_PAGE_SIZE:5000}")
     private var pageSize: Int = 5000
 
-    val CURRENT_VERSION = 2
+    val CURRENT_VERSION = 3
 
     private val logger = LoggerFactory.getLogger(javaClass)
 
@@ -131,10 +131,14 @@ class FixOldScores(
             .set(SCORES.EDGE_HITS, processedReplay.edge_hits)
             .set(SCORES.KEYPRESSES_TIMES, processedReplay.keypresses_times?.toTypedArray())
             .set(SCORES.KEYPRESSES_MEDIAN, processedReplay.keypresses_median)
+            .set(SCORES.KEYPRESSES_MEDIAN_ADJUSTED, processedReplay.keypresses_median_adjusted)
             .set(SCORES.KEYPRESSES_STANDARD_DEVIATION, processedReplay.keypresses_standard_deviation)
+            .set(SCORES.KEYPRESSES_STANDARD_DEVIATION_ADJUSTED, processedReplay.keypresses_standard_deviation_adjusted)
             .set(SCORES.SLIDEREND_RELEASE_TIMES, processedReplay.sliderend_release_times?.toTypedArray())
             .set(SCORES.SLIDEREND_RELEASE_MEDIAN, processedReplay.sliderend_release_median)
+            .set(SCORES.SLIDEREND_RELEASE_MEDIAN_ADJUSTED, processedReplay.sliderend_release_median_adjusted)
             .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION, processedReplay.sliderend_release_standard_deviation)
+            .set(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, processedReplay.sliderend_release_standard_deviation_adjusted)
             .where(SCORES.REPLAY_ID.eq(score.replayId))
             .returningResult(SCORES.ID)
             .fetchOne()?.getValue(SCORES.ID)

nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt

@@ -64,7 +64,7 @@ class ImportScores(
         }
     }
 
-    val CURRENT_VERSION = 2
+    val CURRENT_VERSION = 3
 
     @Value("\${WEBHOOK_URL}")
     private lateinit var webhookUrl: String

nise-backend/src/main/resources/db/migration/V0.0.1.015__alter_scores.sql

@@ -0,0 +1,5 @@
+ALTER TABLE public.scores
+    ADD COLUMN keypresses_median_adjusted float8,
+    ADD COLUMN keypresses_standard_deviation_adjusted float8,
+    ADD COLUMN sliderend_release_median_adjusted float8,
+    ADD COLUMN sliderend_release_standard_deviation_adjusted float8;

nise-circleguard/src/main.py

@@ -10,6 +10,7 @@ import numpy as np
 import scipy
 from brparser import Replay, BeatmapOsu, Mod
 from circleguard import Circleguard, ReplayString, Hit
+from circleguard.utils import filter_outliers
 from flask import Flask, request, jsonify, abort
 
 from src.WriteStreamWrapper import WriteStreamWrapper
@@ -60,11 +61,15 @@ class ReplayResponse:
 
     keypresses_times: List[int]
     keypresses_median: float
+    keypresses_median_adjusted: float
     keypresses_standard_deviation: float
+    keypresses_standard_deviation_adjusted: float
 
     sliderend_release_times: List[int]
     sliderend_release_median: float
+    sliderend_release_median_adjusted: float
     sliderend_release_standard_deviation: float
+    sliderend_release_standard_deviation_adjusted: float
 
     judgements: List[Hit]
 
@@ -174,11 +179,15 @@ def process_replay():
 
             keypresses_times=kp,
             keypresses_median=np.median(kp),
+            keypresses_median_adjusted=np.median(filter_outliers(kp)),
             keypresses_standard_deviation=np.std(kp, ddof=1),
+            keypresses_standard_deviation_adjusted=np.std(filter_outliers(kp), ddof=1),
 
             sliderend_release_times=se,
             sliderend_release_median=np.median(se),
+            sliderend_release_median_adjusted=np.median(filter_outliers(se)),
             sliderend_release_standard_deviation=np.std(se, ddof=1),
+            sliderend_release_standard_deviation_adjusted=np.std(filter_outliers(se), ddof=1),
 
             judgements=judgements
         )