diff --git a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/UpdateUserQueue.kt b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/UpdateUserQueue.kt index 6ea4e02..b7f75b6 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/UpdateUserQueue.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/UpdateUserQueue.kt @@ -18,7 +18,7 @@ import org.jooq.Identity import org.jooq.Name import org.jooq.Record import org.jooq.Records -import org.jooq.Row7 +import org.jooq.Row8 import org.jooq.Schema import org.jooq.SelectField import org.jooq.Table @@ -99,6 +99,11 @@ open class UpdateUserQueue( */ val PROGRESS_TOTAL: TableField = createField(DSL.name("progress_total"), SQLDataType.INTEGER, this, "") + /** + * The column public.update_user_queue.added_by_user_id. + */ + val ADDED_BY_USER_ID: TableField = createField(DSL.name("added_by_user_id"), SQLDataType.BIGINT, this, "") + private constructor(alias: Name, aliased: Table?): this(alias, null, null, aliased, null) private constructor(alias: Name, aliased: Table?, parameters: Array?>?): this(alias, null, null, aliased, parameters) @@ -141,18 +146,18 @@ open class UpdateUserQueue( override fun rename(name: Table<*>): UpdateUserQueue = UpdateUserQueue(name.getQualifiedName(), null) // ------------------------------------------------------------------------- - // Row7 type methods + // Row8 type methods // ------------------------------------------------------------------------- - override fun fieldsRow(): Row7 = super.fieldsRow() as Row7 + override fun fieldsRow(): Row8 = super.fieldsRow() as Row8 /** * Convenience mapping calling {@link SelectField#convertFrom(Function)}. */ - fun mapping(from: (Int?, Long?, Boolean?, LocalDateTime?, OffsetDateTime?, Int?, Int?) -> U): SelectField = convertFrom(Records.mapping(from)) + fun mapping(from: (Int?, Long?, Boolean?, LocalDateTime?, OffsetDateTime?, Int?, Int?, Long?) -> U): SelectField = convertFrom(Records.mapping(from)) /** * Convenience mapping calling {@link SelectField#convertFrom(Class, * Function)}. */ - fun mapping(toType: Class, from: (Int?, Long?, Boolean?, LocalDateTime?, OffsetDateTime?, Int?, Int?) -> U): SelectField = convertFrom(toType, Records.mapping(from)) + fun mapping(toType: Class, from: (Int?, Long?, Boolean?, LocalDateTime?, OffsetDateTime?, Int?, Int?, Long?) -> U): SelectField = convertFrom(toType, Records.mapping(from)) } diff --git a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/UpdateUserQueueRecord.kt b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/UpdateUserQueueRecord.kt index 1345afd..ab2a9f1 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/UpdateUserQueueRecord.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/UpdateUserQueueRecord.kt @@ -11,8 +11,8 @@ import java.time.OffsetDateTime import org.jooq.Field import org.jooq.Record1 -import org.jooq.Record7 -import org.jooq.Row7 +import org.jooq.Record8 +import org.jooq.Row8 import org.jooq.impl.UpdatableRecordImpl @@ -20,7 +20,7 @@ import org.jooq.impl.UpdatableRecordImpl * This class is generated by jOOQ. */ @Suppress("UNCHECKED_CAST") -open class UpdateUserQueueRecord private constructor() : UpdatableRecordImpl(UpdateUserQueue.UPDATE_USER_QUEUE), Record7 { +open class UpdateUserQueueRecord private constructor() : UpdatableRecordImpl(UpdateUserQueue.UPDATE_USER_QUEUE), Record8 { open var id: Int? set(value): Unit = set(0, value) @@ -50,6 +50,10 @@ open class UpdateUserQueueRecord private constructor() : UpdatableRecordImpl = super.key() as Record1 // ------------------------------------------------------------------------- - // Record7 type implementation + // Record8 type implementation // ------------------------------------------------------------------------- - override fun fieldsRow(): Row7 = super.fieldsRow() as Row7 - override fun valuesRow(): Row7 = super.valuesRow() as Row7 + override fun fieldsRow(): Row8 = super.fieldsRow() as Row8 + override fun valuesRow(): Row8 = super.valuesRow() as Row8 override fun field1(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.ID override fun field2(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.USER_ID override fun field3(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.PROCESSED @@ -69,6 +73,7 @@ open class UpdateUserQueueRecord private constructor() : UpdatableRecordImpl = UpdateUserQueue.UPDATE_USER_QUEUE.PROCESSED_AT override fun field6(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.PROGRESS_CURRENT override fun field7(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.PROGRESS_TOTAL + override fun field8(): Field = UpdateUserQueue.UPDATE_USER_QUEUE.ADDED_BY_USER_ID override fun component1(): Int? = id override fun component2(): Long = userId override fun component3(): Boolean? = processed @@ -76,6 +81,7 @@ open class UpdateUserQueueRecord private constructor() : UpdatableRecordImpl csrf.disable() } .addFilterBefore(customCorsFilter(), CorsFilter::class.java) + .authorizeHttpRequests { auth -> + auth + .requestMatchers("/user-queue").authenticated() + .anyRequest().permitAll() + } .oauth2Login { oauthLogin -> oauthLogin.successHandler(CustomAuthenticationSuccessHandler()) } diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UserDetailsController.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UserDetailsController.kt index 09d61f5..9c31733 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UserDetailsController.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/controller/UserDetailsController.kt @@ -7,6 +7,7 @@ import com.nisemoe.nise.SuspiciousScoreEntry import com.nisemoe.nise.UserDetails import com.nisemoe.nise.UserQueueDetails import com.nisemoe.nise.database.UserService +import com.nisemoe.nise.service.AuthService import com.nisemoe.nise.service.UpdateUserQueueService import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping @@ -21,6 +22,7 @@ import java.time.ZoneOffset @RestController class UserDetailsController( private val scoreService: ScoreService, + private val authService: AuthService, private val userService: UserService, private val userQueueService: UpdateUserQueueService ) { @@ -43,7 +45,8 @@ class UserDetailsController( if(!userQueueDetails.canUpdate) return ResponseEntity.badRequest().build() - val inserted = this.userQueueService.insertUser(request.userId) + val inserted = this.userQueueService.insertUser(request.userId, addedByUserId = this.authService.getCurrentUser().userId) + return if(inserted) ResponseEntity.ok().build() else diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt index e0e5f96..dd9db3f 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt @@ -154,12 +154,14 @@ class OsuApi( } } - fun getTopUserScores(userId: Long): List? { + fun getTopUserScores(userId: Long, type: String = "best"): List? { val queryParams = mapOf( "mode" to "osu", - "limit" to "100" + "limit" to "100", + "legacy_only" to "1" ) - val response = doRequest("https://osu.ppy.sh/api/v2/users/$userId/scores/best?", queryParams) + + val response = doRequest("https://osu.ppy.sh/api/v2/users/$userId/scores/$type?", queryParams) if(response == null) { this.logger.info("Error loading top user scores") return null diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt index 4df4c35..f05f39e 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt @@ -72,7 +72,7 @@ class ImportScores( private lateinit var webhookUrl: String private val logger = LoggerFactory.getLogger(javaClass) - + private final val sleepTimeInMs = 500L private final val UPDATE_USER_EVERY_DAYS = 7L @@ -157,76 +157,100 @@ class ImportScores( for(userId in queue) { val topUserScores = this.osuApi.getTopUserScores(userId = userId) + val recentUserScores = this.osuApi.getTopUserScores(userId = userId, type = "recent") + val firstPlaceUserScores = this.osuApi.getTopUserScores(userId = userId, type = "firsts") + + this.logger.info("Processing user with id = $userId") + this.logger.info("Top scores: ${topUserScores?.size}") + this.logger.info("Recent scores: ${recentUserScores?.size}") + this.logger.info("First place scores: ${firstPlaceUserScores?.size}") + Thread.sleep(this.sleepTimeInMs) - if(topUserScores != null) { - val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS))) - if(!userExists) { - val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id") - if(apiUser != null) { - this.userService.insertApiUser(apiUser) - this.statistics.usersAddedToDatabase++ - } else { - this.logger.error("Failed to fetch user with id = $userId") - } - } + if(topUserScores == null || recentUserScores == null || firstPlaceUserScores == null) { + this.logger.error("Failed to fetch top scores for user with id = $userId") + continue + } - var current = 0 + val allUserScores = (topUserScores + recentUserScores + firstPlaceUserScores) + .filter { it.beatmap != null && it.beatmapset != null } + .distinctBy { it.best_id } - val lastCompletedUpdate = dslContext.select(UPDATE_USER_QUEUE.PROCESSED_AT) - .from(UPDATE_USER_QUEUE) - .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) - .and(UPDATE_USER_QUEUE.PROCESSED.isTrue) - .orderBy(UPDATE_USER_QUEUE.PROCESSED_AT.desc()) - .limit(1) - .fetchOneInto(OffsetDateTime::class.java) + this.logger.info("Unique scores: ${allUserScores.size}") - for(topScore in topUserScores) { - if(topScore.beatmap != null && topScore.beatmapset != null) { - val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap.id)) - if (!beatmapExists) { - dslContext.insertInto(BEATMAPS) - .set(BEATMAPS.BEATMAP_ID, topScore.beatmap.id) - .set(BEATMAPS.BEATMAPSET_ID, topScore.beatmapset.id) - .set(BEATMAPS.STAR_RATING, topScore.beatmap.difficulty_rating) - .set(BEATMAPS.VERSION, topScore.beatmap.version) - .set(BEATMAPS.ARTIST, topScore.beatmapset.artist) - .set(BEATMAPS.SOURCE, topScore.beatmapset.source) - .set(BEATMAPS.TITLE, topScore.beatmapset.title) - .set(BEATMAPS.SOURCE, topScore.beatmapset.source) - .set(BEATMAPS.CREATOR, topScore.beatmapset.creator) - .execute() - this.statistics.beatmapsAddedToDatabase++ - } - - // Update the database - dslContext.update(UPDATE_USER_QUEUE) - .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current) - .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, topUserScores.size) - .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) - .and(UPDATE_USER_QUEUE.PROCESSED.isFalse) - .execute() - - val currentQueueDetails = UserQueueDetails( - isProcessing = true, - lastCompletedUpdate = lastCompletedUpdate, - canUpdate = false, - progressCurrent = current, - progressTotal = topUserScores.size - ) - - // Update the frontend - messagingTemplate.convertAndSend( - "/topic/live-user/${userId}", - UpdateUserQueueService.UserQueueWebsocketPacket(message = "UPDATE_PROGRESS", data = currentQueueDetails) - ) - - this.insertAndProcessNewScore(topScore.beatmap.id, topScore, isUserQueue = true) - } - current += 1 + val userExists = dslContext.fetchExists(USERS, USERS.USER_ID.eq(userId), USERS.SYS_LAST_UPDATE.greaterOrEqual(OffsetDateTime.now(ZoneOffset.UTC).minusDays(UPDATE_USER_EVERY_DAYS))) + if(!userExists) { + val apiUser = this.osuApi.getUserProfile(userId = userId.toString(), mode = "osu", key = "id") + if(apiUser != null) { + this.userService.insertApiUser(apiUser) + this.statistics.usersAddedToDatabase++ + } else { + this.logger.error("Failed to fetch user with id = $userId") } } + var current = 0 + + val lastCompletedUpdate = dslContext.select(UPDATE_USER_QUEUE.PROCESSED_AT) + .from(UPDATE_USER_QUEUE) + .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) + .and(UPDATE_USER_QUEUE.PROCESSED.isTrue) + .orderBy(UPDATE_USER_QUEUE.PROCESSED_AT.desc()) + .limit(1) + .fetchOneInto(OffsetDateTime::class.java) + + for(topScore in allUserScores) { + val beatmapExists = dslContext.fetchExists(BEATMAPS, BEATMAPS.BEATMAP_ID.eq(topScore.beatmap!!.id)) + if (!beatmapExists) { + dslContext.insertInto(BEATMAPS) + .set(BEATMAPS.BEATMAP_ID, topScore.beatmap.id) + .set(BEATMAPS.BEATMAPSET_ID, topScore.beatmapset!!.id) + .set(BEATMAPS.STAR_RATING, topScore.beatmap.difficulty_rating) + .set(BEATMAPS.VERSION, topScore.beatmap.version) + .set(BEATMAPS.ARTIST, topScore.beatmapset.artist) + .set(BEATMAPS.SOURCE, topScore.beatmapset.source) + .set(BEATMAPS.TITLE, topScore.beatmapset.title) + .set(BEATMAPS.SOURCE, topScore.beatmapset.source) + .set(BEATMAPS.CREATOR, topScore.beatmapset.creator) + .execute() + this.statistics.beatmapsAddedToDatabase++ + } + + // Update the database + dslContext.update(UPDATE_USER_QUEUE) + .set(UPDATE_USER_QUEUE.PROGRESS_CURRENT, current) + .set(UPDATE_USER_QUEUE.PROGRESS_TOTAL, allUserScores.size) + .where(UPDATE_USER_QUEUE.USER_ID.eq(userId)) + .and(UPDATE_USER_QUEUE.PROCESSED.isFalse) + .execute() + + val currentQueueDetails = UserQueueDetails( + isProcessing = true, + lastCompletedUpdate = lastCompletedUpdate, + canUpdate = false, + progressCurrent = current, + progressTotal = allUserScores.size + ) + + // Update the frontend + messagingTemplate.convertAndSend( + "/topic/live-user/${userId}", + UpdateUserQueueService.UserQueueWebsocketPacket(message = "UPDATE_PROGRESS", data = currentQueueDetails) + ) + + this.insertAndProcessNewScore(topScore.beatmap.id, topScore, isUserQueue = true) + + current += 1 + } + + // Check for stolen replays. + val uniqueBeatmapIds = allUserScores + .groupBy { it.beatmap!!.id } + + this.logger.info("Checking similarity for ${uniqueBeatmapIds.size} beatmaps.") + + uniqueBeatmapIds.forEach { checkReplaySimilarity(it.key) } + // Update the backend this.updateUserQueueService.setUserAsProcessed(userId) } @@ -432,6 +456,11 @@ class ImportScores( .and(SCORES.IS_BANNED.isFalse) .fetchInto(ReplayDto::class.java) + if(allReplays.size < 2) { + this.logger.debug("Not enough replays to compare for beatmapId = $beatmapId.") + return + } + sw.start("konata") val konataResults: List = try { diff --git a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt index 6306626..b1c6800 100644 --- a/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt +++ b/nise-backend/src/main/kotlin/com/nisemoe/nise/service/UpdateUserQueueService.kt @@ -91,7 +91,7 @@ class UpdateUserQueueService( * Inserts a user into the update queue if they are not already in the queue. * @return true if the user was inserted, false if the user was already in the queue */ - fun insertUser(userId: Long): Boolean { + fun insertUser(userId: Long, addedByUserId: Long? = null): Boolean { val exists = dslContext.fetchExists(UPDATE_USER_QUEUE, UPDATE_USER_QUEUE.USER_ID.eq(userId), UPDATE_USER_QUEUE.PROCESSED.isFalse @@ -101,6 +101,7 @@ class UpdateUserQueueService( val insertedRows = dslContext.insertInto(UPDATE_USER_QUEUE) .set(UPDATE_USER_QUEUE.USER_ID, userId) + .set(UPDATE_USER_QUEUE.ADDED_BY_USER_ID, addedByUserId) .execute() return insertedRows == 1 diff --git a/nise-backend/src/main/resources/db/migration/V0.0.1.021__alter_users_queue.sql b/nise-backend/src/main/resources/db/migration/V0.0.1.021__alter_users_queue.sql new file mode 100644 index 0000000..ae35a17 --- /dev/null +++ b/nise-backend/src/main/resources/db/migration/V0.0.1.021__alter_users_queue.sql @@ -0,0 +1,2 @@ +ALTER TABLE public.update_user_queue + ADD COLUMN added_by_user_id bigint; \ No newline at end of file diff --git a/nise-frontend/src/app/view-user/view-user.component.html b/nise-frontend/src/app/view-user/view-user.component.html index 88e6a61..5f52ce0 100644 --- a/nise-frontend/src/app/view-user/view-user.component.html +++ b/nise-frontend/src/app/view-user/view-user.component.html @@ -40,29 +40,31 @@
  • Playcount: {{ this.userInfo.user_details.playcount | number: '1.0-1' }}
  • - - - - update user scores now! - - - - wait a bit to force update - - | - last update: {{ this.userInfo.queue_details.lastCompletedUpdate ? this.calculateTimeAgo(this.userInfo.queue_details.lastCompletedUpdate) : 'never'}} - - -
    - updating now! | progress: {{ this.userInfo.queue_details.progressCurrent != null ? this.userInfo.queue_details.progressCurrent : "?" }}/{{ this.userInfo.queue_details.progressTotal != null ? this.userInfo.queue_details.progressTotal : "?" }} - - (in queue, be patient) + + + + + update user scores now! + - - - -
    -
    + + wait a bit to force update + + | + last update: {{ this.userInfo.queue_details.lastCompletedUpdate ? this.calculateTimeAgo(this.userInfo.queue_details.lastCompletedUpdate) : 'never'}} + + +
    + updating now! | progress: {{ this.userInfo.queue_details.progressCurrent != null ? this.userInfo.queue_details.progressCurrent : "?" }}/{{ this.userInfo.queue_details.progressTotal != null ? this.userInfo.queue_details.progressTotal : "?" }} + + (in queue, be patient) + + + + +
    +
    +

    Suspicious Scores ({{ this.userInfo.suspicious_scores.length }})

    diff --git a/nise-frontend/src/app/view-user/view-user.component.ts b/nise-frontend/src/app/view-user/view-user.component.ts index 85d4d54..d30259c 100644 --- a/nise-frontend/src/app/view-user/view-user.component.ts +++ b/nise-frontend/src/app/view-user/view-user.component.ts @@ -13,6 +13,7 @@ import {Message} from "@stomp/stompjs/esm6"; import {CuteLoadingComponent} from "../../corelib/components/cute-loading/cute-loading.component"; import {differenceInDays, differenceInHours} from "date-fns/fp"; import {FilterManagerService} from "../filter-manager.service"; +import {UserService} from "../../corelib/service/user.service"; interface UserInfo { user_details: UserDetails; @@ -63,6 +64,7 @@ export class ViewUserComponent implements OnInit, OnChanges, OnDestroy { private activatedRoute: ActivatedRoute, private title: Title, private rxStompService: RxStompService, + public userService: UserService ) { } getUserInfo(): Observable {