From 95650c5ca66ffe47f6b1e5cffb39c9033650ffbd Mon Sep 17 00:00:00 2001 From: "nise.moe" Date: Fri, 16 Feb 2024 15:19:36 +0100 Subject: [PATCH 1/2] Basic agent implementation --- .../com/nisemoe/nise/scheduler/Agent.kt | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt new file mode 100644 index 0000000..fff0d10 --- /dev/null +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt @@ -0,0 +1,96 @@ +package com.nisemoe.nise.scheduler + +import com.nisemoe.generated.tables.records.ScoresRecord +import com.nisemoe.generated.tables.references.SCORES +import com.nisemoe.generated.tables.references.USERS +import org.jooq.DSLContext +import org.springframework.context.annotation.Profile +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service + +data class UserReport( + val username: String, + val susScore: Double, + val userId: Long, + val userScores: List +) + +@Service +@Profile("agent") +class Agent( + private val dslContext: DSLContext +) { + + // 20 minutes to ms = 1200000 + @Scheduled(fixedDelay = 1200000, initialDelay = 0) + fun buildReports() { + + // Select sus scores + val susUsers = dslContext.select( + SCORES.USER_ID, + SCORES.ADJUSTED_UR, + SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, + SCORES.PP, + SCORES.EDGE_HITS, + SCORES.SNAPS, + SCORES.KEYPRESSES_MEDIAN_ADJUSTED, + SCORES.ERROR_KURTOSIS + ) + .from(SCORES) + .where(SCORES.ADJUSTED_UR.lessOrEqual(10.0)) + .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.lessOrEqual(8.0)) + .and(SCORES.IS_BANNED.isFalse) + .and(SCORES.VERSION.greaterOrEqual(4)) + .fetchInto(ScoresRecord::class.java) + + val reports = mutableListOf() + + for(user in susUsers.groupBy { it.userId }) { + val userId = user.key!! + val username = dslContext.select(USERS.USERNAME) + .from(USERS) + .where(USERS.USER_ID.eq(userId)) + .fetchOne(USERS.USERNAME) ?: continue + + val userScores = user.value + val susScore = calculateSuspiciousnessScore(userScores) + + reports.add(UserReport(userId = userId, username = username, userScores = userScores, susScore = susScore)) + } + + reports.sortByDescending { it.susScore } + + println(reports) + } + + fun calculateSuspiciousnessScore(scores: List): Double { + if (scores.isEmpty()) return 0.0 + + return scores.sumOf { score -> + var suspiciousnessScore = 0.0 + + suspiciousnessScore += score.pp!! * 1.0 + + suspiciousnessScore += if (score.adjustedUr!! < 6.0) 5.0 else 0.0 + + suspiciousnessScore += if (score.sliderendReleaseStandardDeviationAdjusted!! < 6) 5.0 else 0.0 + + suspiciousnessScore += when { + score.edgeHits!! > 4 -> (score.edgeHits!! - 4) * 2.0 + else -> 0.0 + } + + suspiciousnessScore += when { + score.snaps!! > 4 -> (score.snaps!! - 4) * 2.0 + else -> 0.0 + } + + suspiciousnessScore += if (score.keypressesMedianAdjusted!! < 6) 5.0 else 0.0 + + suspiciousnessScore += if (score.errorKurtosis!! < 6) 5.0 else 0.0 + + suspiciousnessScore + } + } + +} \ No newline at end of file From 00676423593ae6a3281646a270bc92c369824ce0 Mon Sep 17 00:00:00 2001 From: "nise.moe" Date: Fri, 16 Feb 2024 17:47:45 +0100 Subject: [PATCH 2/2] More work on getting a basic agent implementation to work --- .../com/nisemoe/nise/scheduler/Agent.kt | 220 ++++++++++++++---- .../com/nisemoe/nise/scheduler/AgentTest.kt | 28 +++ 2 files changed, 200 insertions(+), 48 deletions(-) create mode 100644 nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/AgentTest.kt diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt index fff0d10..9c44515 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/Agent.kt @@ -1,96 +1,220 @@ package com.nisemoe.nise.scheduler import com.nisemoe.generated.tables.records.ScoresRecord +import com.nisemoe.generated.tables.records.UsersRecord import com.nisemoe.generated.tables.references.SCORES import com.nisemoe.generated.tables.references.USERS +import com.nisemoe.nise.osu.OsuApi import org.jooq.DSLContext import org.springframework.context.annotation.Profile import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.time.temporal.ChronoUnit +import kotlin.math.roundToInt data class UserReport( val username: String, val susScore: Double, + val urgencyScore: Double, val userId: Long, - val userScores: List -) + val userScores: List +) { + + override fun toString(): String { + val s = "User: $username (https://nise.moe/u/${username})- Suspiciousness score: ${susScore.roundToInt()} | Urgency score: ${urgencyScore.roundToInt()}" + val s2 = "Average PP: ${userScores.map { it.pp }.average()} | Average UR: ${userScores.map { it.adjustedUr }.average()} | Average Sliderend Release: ${userScores.map { it.sliderendReleaseStandardDeviationAdjusted }.average()}" + return s + "\n" + s2 + "\n" + } + +} @Service @Profile("agent") class Agent( - private val dslContext: DSLContext + private val dslContext: DSLContext, + private val osuApi: OsuApi ) { - // 20 minutes to ms = 1200000 @Scheduled(fixedDelay = 1200000, initialDelay = 0) fun buildReports() { - - // Select sus scores - val susUsers = dslContext.select( - SCORES.USER_ID, - SCORES.ADJUSTED_UR, - SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, - SCORES.PP, - SCORES.EDGE_HITS, - SCORES.SNAPS, - SCORES.KEYPRESSES_MEDIAN_ADJUSTED, - SCORES.ERROR_KURTOSIS - ) - .from(SCORES) - .where(SCORES.ADJUSTED_UR.lessOrEqual(10.0)) - .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.lessOrEqual(8.0)) - .and(SCORES.IS_BANNED.isFalse) - .and(SCORES.VERSION.greaterOrEqual(4)) - .fetchInto(ScoresRecord::class.java) - val reports = mutableListOf() - for(user in susUsers.groupBy { it.userId }) { - val userId = user.key!! - val username = dslContext.select(USERS.USERNAME) - .from(USERS) + val userIds = dslContext.select(SCORES.USER_ID) + .from(SCORES) + .where(SCORES.ADJUSTED_UR.lessOrEqual(256.0)) + .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.isNotNull) + .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.lessOrEqual(16.0)) + .and(SCORES.IS_BANNED.isFalse) + .and(SCORES.VERSION.greaterOrEqual(4)) + .groupBy(SCORES.USER_ID) + .fetchInto(Long::class.java) + + println("Found ${userIds.size} users to check.") + + // We check if any of them are banned. +// val bannedIds = mutableListOf() +// for(chunk in userIds.chunked(50)) { +// +// val result = osuApi.getUsersBatch(chunk) +// if(result == null) { +// println("Failed to get users from osu! api") +// return +// } +// +// // Compare which userIds aren't in the result and mark them as banned. +// val bannedUsers = chunk.filter { it !in result.users.map { it.id } } +// bannedIds.addAll(bannedUsers) +// } +// +// if(bannedIds.isNotEmpty()) { +// // Remove all banned users from the list +// susUsers.removeIf { it.userId in bannedIds } +// +// // Mark all their scores as banned +// dslContext.update(SCORES) +// .set(SCORES.IS_BANNED, true) +// .where(SCORES.USER_ID.`in`(bannedIds)) +// .execute() +// +// println("Marked ${bannedIds.size} users as banned: $bannedIds") +// } + + for(userId in userIds) { + val userDetails = dslContext.selectFrom(USERS) .where(USERS.USER_ID.eq(userId)) - .fetchOne(USERS.USERNAME) ?: continue + .fetchOneInto(UsersRecord::class.java) ?: continue - val userScores = user.value - val susScore = calculateSuspiciousnessScore(userScores) + val username = userDetails.username!! + val userScores = dslContext.select( + SCORES.DATE, + SCORES.ADJUSTED_UR, + SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED, + SCORES.PP, + SCORES.EDGE_HITS, + SCORES.SNAPS, + SCORES.KEYPRESSES_MEDIAN_ADJUSTED, + SCORES.ERROR_KURTOSIS + ) + .from(SCORES) + .where(SCORES.ADJUSTED_UR.lessOrEqual(256.0)) + .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.isNotNull) + .and(SCORES.SLIDEREND_RELEASE_STANDARD_DEVIATION_ADJUSTED.lessOrEqual(16.0)) + .and(SCORES.IS_BANNED.isFalse) + .and(SCORES.VERSION.greaterOrEqual(4)) + .and(SCORES.USER_ID.eq(userId)) + .fetchInto(ScoresRecord::class.java) - reports.add(UserReport(userId = userId, username = username, userScores = userScores, susScore = susScore)) + if(areMetricsBlatant(userScores.map { mapScoreToMetrics(userDetails, it) })) + continue + + val susScore = calculateSuspiciousnessScore(userScores.map { mapScoreToMetrics(userDetails, it) }) + + val newReport = UserReport( + userId = userId, + username = username, + userScores = userScores.map { mapScoreToMetrics(userDetails, it) }, + susScore = susScore.first, + urgencyScore = susScore.second + ) + reports.add(newReport) + println(newReport) } - reports.sortByDescending { it.susScore } - - println(reports) + reports.sortByDescending { it.urgencyScore } } - fun calculateSuspiciousnessScore(scores: List): Double { - if (scores.isEmpty()) return 0.0 + fun mapScoreToMetrics(user: UsersRecord, score: ScoresRecord): ScoreMetrics { + return ScoreMetrics( + date = score.date!!, + country = user.country!!, + pp = score.pp!!, + adjustedUr = score.adjustedUr!!, + sliderendReleaseStandardDeviationAdjusted = score.sliderendReleaseStandardDeviationAdjusted!!, + edgeHits = score.edgeHits!!, + snaps = score.snaps!!, + keypressesMedianAdjusted = score.keypressesMedianAdjusted!!, + errorKurtosis = score.errorKurtosis!! + ) + } - return scores.sumOf { score -> - var suspiciousnessScore = 0.0 + data class ScoreMetrics( + val date: LocalDateTime, + val country: String, + val pp: Double, + val adjustedUr: Double, + val sliderendReleaseStandardDeviationAdjusted: Double, + val edgeHits: Int, + val snaps: Int, + val keypressesMedianAdjusted: Double, + val errorKurtosis: Double + ) - suspiciousnessScore += score.pp!! * 1.0 + val ppWeight = 1.0 + val urWeight = 2.0 + val sliderendReleaseWeight = 2.0 + val edgeHitsWeight = 2.0 + val snapsWeight = 2.0 + val keypressesWeight = 2.0 + val errorKurtosisWeight = 1.0 + val recencyBoost = 10.0 + val spreadBoost = 5.0 - suspiciousnessScore += if (score.adjustedUr!! < 6.0) 5.0 else 0.0 + fun areMetricsBlatant(metrics: List): Boolean { + return metrics.map { it.adjustedUr }.average() < 24.0 + } - suspiciousnessScore += if (score.sliderendReleaseStandardDeviationAdjusted!! < 6) 5.0 else 0.0 + fun calculateSuspiciousnessScore(scores: List): Pair { + if (scores.isEmpty()) return Pair(0.0, 0.0) - suspiciousnessScore += when { - score.edgeHits!! > 4 -> (score.edgeHits!! - 4) * 2.0 + val currentDate = LocalDateTime.now() + val mostRecentScoreDate = scores.maxByOrNull { it.date }?.date ?: return Pair(0.0, 0.0) + val daysSinceMostRecentScore = ChronoUnit.DAYS.between(mostRecentScoreDate, currentDate) + val recencyScore = if (daysSinceMostRecentScore <= 30) recencyBoost else 0.0 + + val dateSpread = scores.map { score -> ChronoUnit.MONTHS.between(score.date, currentDate) }.average() + val spreadScore = if (dateSpread > 12) spreadBoost else 0.0 + + val (suspiciousnessScore, ppTotal) = scores.fold(Pair(0.0, 0.0)) { acc, score -> + var suspiciousness = acc.first + var ppSum = acc.second + + suspiciousness += score.pp * ppWeight + + if (score.adjustedUr < 6.0) { + suspiciousness += (6.0 - score.adjustedUr) * urWeight + } + + if (score.sliderendReleaseStandardDeviationAdjusted < 6.0) { + suspiciousness += (6.0 - score.sliderendReleaseStandardDeviationAdjusted) * sliderendReleaseWeight + } + + suspiciousness += when { + score.edgeHits > 4 -> (score.edgeHits - 4) * edgeHitsWeight else -> 0.0 } - suspiciousnessScore += when { - score.snaps!! > 4 -> (score.snaps!! - 4) * 2.0 + suspiciousness += when { + score.snaps > 4 -> (score.snaps - 4) * snapsWeight else -> 0.0 } - suspiciousnessScore += if (score.keypressesMedianAdjusted!! < 6) 5.0 else 0.0 + if (score.keypressesMedianAdjusted < 6) { + suspiciousness += (6.0 - score.keypressesMedianAdjusted) * keypressesWeight + } - suspiciousnessScore += if (score.errorKurtosis!! < 6) 5.0 else 0.0 + if (score.errorKurtosis <= 0.0) { + suspiciousness += (6.0 - score.errorKurtosis) * errorKurtosisWeight + } - suspiciousnessScore + ppSum += score.pp + Pair(suspiciousness, ppSum) } + + val averageSuspiciousnessScore = suspiciousnessScore / scores.size + val urgencyScore = ppTotal / scores.size + recencyScore + spreadScore + + return Pair(averageSuspiciousnessScore, urgencyScore) } } \ No newline at end of file diff --git a/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/AgentTest.kt b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/AgentTest.kt new file mode 100644 index 0000000..95c17a3 --- /dev/null +++ b/nise-backend/src/test/kotlin/com/nisemoe/nise/scheduler/AgentTest.kt @@ -0,0 +1,28 @@ +package com.nisemoe.nise.scheduler + +import com.nisemoe.nise.database.UserService +import com.nisemoe.nise.osu.OsuApi +import com.nisemoe.nise.osu.TokenService +import com.nisemoe.nise.service.CacheService +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.annotation.Import +import org.springframework.test.context.ActiveProfiles + +@SpringBootTest +@ActiveProfiles("postgres", "agent") +@MockBean(GlobalCache::class, UserService::class) +@Import(Agent::class, OsuApi::class, TokenService::class, CacheService::class) +class AgentTest { + + @Autowired + lateinit var agent: Agent + + @Test + fun buildReports() { + agent.buildReports() + } + +} \ No newline at end of file