Stedos/nise
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Efficiently cycle trough a list of osu api keys, saved in the database, to avoid rate limits

Browse Source
This commit is contained in:
nise.moe 2024-02-22 16:26:41 +01:00
parent f3d8b69166
commit c78faf18f2
11 changed files with 384 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nise-backend/src/main/kotlin/com/nisemoe/generated/Public.kt
View File

@ -16,6 +16,7 @@ import com.nisemoe.generated.sequences.USERS_USER_ID_SEQ
import com.nisemoe.generated.sequences.USERS_USER_ID_SEQ1
import com.nisemoe.generated.tables.Beatmaps
import com.nisemoe.generated.tables.FlywaySchemaHistory
import com.nisemoe.generated.tables.OsuApiKeys
import com.nisemoe.generated.tables.RedditPost
import com.nisemoe.generated.tables.Scores
import com.nisemoe.generated.tables.ScoresJudgements
@ -54,6 +55,11 @@ open class Public : SchemaImpl("public", DefaultCatalog.DEFAULT_CATALOG) {
*/
val FLYWAY_SCHEMA_HISTORY: FlywaySchemaHistory get() = FlywaySchemaHistory.FLYWAY_SCHEMA_HISTORY
/**
* The table <code>public.osu_api_keys</code>.
*/
val OSU_API_KEYS: OsuApiKeys get() = OsuApiKeys.OSU_API_KEYS
/**
* The table <code>public.reddit_post</code>.
*/
@ -102,6 +108,7 @@ open class Public : SchemaImpl("public", DefaultCatalog.DEFAULT_CATALOG) {
override fun getTables(): List<Table<*>> = listOf(
Beatmaps.BEATMAPS,
FlywaySchemaHistory.FLYWAY_SCHEMA_HISTORY,
OsuApiKeys.OSU_API_KEYS,
RedditPost.REDDIT_POST,
Scores.SCORES,
ScoresJudgements.SCORES_JUDGEMENTS,

3
nise-backend/src/main/kotlin/com/nisemoe/generated/keys/Keys.kt
View File

@ -6,6 +6,7 @@ package com.nisemoe.generated.keys
import com.nisemoe.generated.tables.Beatmaps
import com.nisemoe.generated.tables.FlywaySchemaHistory
import com.nisemoe.generated.tables.OsuApiKeys
import com.nisemoe.generated.tables.RedditPost
import com.nisemoe.generated.tables.Scores
import com.nisemoe.generated.tables.ScoresJudgements
@ -14,6 +15,7 @@ import com.nisemoe.generated.tables.UpdateUserQueue
import com.nisemoe.generated.tables.Users
import com.nisemoe.generated.tables.records.BeatmapsRecord
import com.nisemoe.generated.tables.records.FlywaySchemaHistoryRecord
import com.nisemoe.generated.tables.records.OsuApiKeysRecord
import com.nisemoe.generated.tables.records.RedditPostRecord
import com.nisemoe.generated.tables.records.ScoresJudgementsRecord
import com.nisemoe.generated.tables.records.ScoresRecord
@ -34,6 +36,7 @@ import org.jooq.impl.Internal
val BEATMAPS_PKEY: UniqueKey<BeatmapsRecord> = Internal.createUniqueKey(Beatmaps.BEATMAPS, DSL.name("beatmaps_pkey"), arrayOf(Beatmaps.BEATMAPS.BEATMAP_ID), true)
val FLYWAY_SCHEMA_HISTORY_PK: UniqueKey<FlywaySchemaHistoryRecord> = Internal.createUniqueKey(FlywaySchemaHistory.FLYWAY_SCHEMA_HISTORY, DSL.name("flyway_schema_history_pk"), arrayOf(FlywaySchemaHistory.FLYWAY_SCHEMA_HISTORY.INSTALLED_RANK), true)
val OSU_API_KEYS_PKEY: UniqueKey<OsuApiKeysRecord> = Internal.createUniqueKey(OsuApiKeys.OSU_API_KEYS, DSL.name("osu_api_keys_pkey"), arrayOf(OsuApiKeys.OSU_API_KEYS.ID), true)
val REDDIT_POST_PKEY: UniqueKey<RedditPostRecord> = Internal.createUniqueKey(RedditPost.REDDIT_POST, DSL.name("reddit_post_pkey"), arrayOf(RedditPost.REDDIT_POST.POST_ID), true)
val REPLAY_ID_UNIQUE: UniqueKey<ScoresRecord> = Internal.createUniqueKey(Scores.SCORES, DSL.name("replay_id_unique"), arrayOf(Scores.SCORES.REPLAY_ID), true)
val SCORES_PKEY: UniqueKey<ScoresRecord> = Internal.createUniqueKey(Scores.SCORES, DSL.name("scores_pkey"), arrayOf(Scores.SCORES.ID), true)

147
nise-backend/src/main/kotlin/com/nisemoe/generated/tables/OsuApiKeys.kt Normal file
View File

@ -0,0 +1,147 @@
/*
* This file is generated by jOOQ.
*/
package com.nisemoe.generated.tables
import com.nisemoe.generated.Public
import com.nisemoe.generated.keys.OSU_API_KEYS_PKEY
import com.nisemoe.generated.tables.records.OsuApiKeysRecord
import java.time.OffsetDateTime
import java.util.function.Function
import org.jooq.Field
import org.jooq.ForeignKey
import org.jooq.Identity
import org.jooq.Name
import org.jooq.Record
import org.jooq.Records
import org.jooq.Row5
import org.jooq.Schema
import org.jooq.SelectField
import org.jooq.Table
import org.jooq.TableField
import org.jooq.TableOptions
import org.jooq.UniqueKey
import org.jooq.impl.DSL
import org.jooq.impl.Internal
import org.jooq.impl.SQLDataType
import org.jooq.impl.TableImpl
/**
* This class is generated by jOOQ.
*/
@Suppress("UNCHECKED_CAST")
open class OsuApiKeys(
alias: Name,
child: Table<out Record>?,
path: ForeignKey<out Record, OsuApiKeysRecord>?,
aliased: Table<OsuApiKeysRecord>?,
parameters: Array<Field<*>?>?
): TableImpl<OsuApiKeysRecord>(
alias,
Public.PUBLIC,
child,
path,
aliased,
parameters,
DSL.comment(""),
TableOptions.table()
) {
companion object {
/**
* The reference instance of <code>public.osu_api_keys</code>
*/
val OSU_API_KEYS: OsuApiKeys = OsuApiKeys()
}
/**
* The class holding records for this type
*/
override fun getRecordType(): Class<OsuApiKeysRecord> = OsuApiKeysRecord::class.java
/**
* The column <code>public.osu_api_keys.id</code>.
*/
val ID: TableField<OsuApiKeysRecord, Int?> = createField(DSL.name("id"), SQLDataType.INTEGER.nullable(false).identity(true), this, "")
/**
* The column <code>public.osu_api_keys.api_key</code>.
*/
val API_KEY: TableField<OsuApiKeysRecord, String?> = createField(DSL.name("api_key"), SQLDataType.CLOB.nullable(false), this, "")
/**
* The column <code>public.osu_api_keys.created_at</code>.
*/
val CREATED_AT: TableField<OsuApiKeysRecord, OffsetDateTime?> = createField(DSL.name("created_at"), SQLDataType.TIMESTAMPWITHTIMEZONE(6).nullable(false).defaultValue(DSL.field(DSL.raw("CURRENT_TIMESTAMP"), SQLDataType.TIMESTAMPWITHTIMEZONE)), this, "")
/**
* The column <code>public.osu_api_keys.is_valid</code>.
*/
val IS_VALID: TableField<OsuApiKeysRecord, Boolean?> = createField(DSL.name("is_valid"), SQLDataType.BOOLEAN.nullable(false).defaultValue(DSL.field(DSL.raw("true"), SQLDataType.BOOLEAN)), this, "")
/**
* The column <code>public.osu_api_keys.is_active</code>.
*/
val IS_ACTIVE: TableField<OsuApiKeysRecord, Boolean?> = createField(DSL.name("is_active"), SQLDataType.BOOLEAN.nullable(false).defaultValue(DSL.field(DSL.raw("true"), SQLDataType.BOOLEAN)), this, "")
private constructor(alias: Name, aliased: Table<OsuApiKeysRecord>?): this(alias, null, null, aliased, null)
private constructor(alias: Name, aliased: Table<OsuApiKeysRecord>?, parameters: Array<Field<*>?>?): this(alias, null, null, aliased, parameters)
/**
* Create an aliased <code>public.osu_api_keys</code> table reference
*/
constructor(alias: String): this(DSL.name(alias))
/**
* Create an aliased <code>public.osu_api_keys</code> table reference
*/
constructor(alias: Name): this(alias, null)
/**
* Create a <code>public.osu_api_keys</code> table reference
*/
constructor(): this(DSL.name("osu_api_keys"), null)
constructor(child: Table<out Record>, key: ForeignKey<out Record, OsuApiKeysRecord>): this(Internal.createPathAlias(child, key), child, key, OSU_API_KEYS, null)
override fun getSchema(): Schema? = if (aliased()) null else Public.PUBLIC
override fun getIdentity(): Identity<OsuApiKeysRecord, Int?> = super.getIdentity() as Identity<OsuApiKeysRecord, Int?>
override fun getPrimaryKey(): UniqueKey<OsuApiKeysRecord> = OSU_API_KEYS_PKEY
override fun `as`(alias: String): OsuApiKeys = OsuApiKeys(DSL.name(alias), this)
override fun `as`(alias: Name): OsuApiKeys = OsuApiKeys(alias, this)
override fun `as`(alias: Table<*>): OsuApiKeys = OsuApiKeys(alias.getQualifiedName(), this)
/**
* Rename this table
*/
override fun rename(name: String): OsuApiKeys = OsuApiKeys(DSL.name(name), null)
/**
* Rename this table
*/
override fun rename(name: Name): OsuApiKeys = OsuApiKeys(name, null)
/**
* Rename this table
*/
override fun rename(name: Table<*>): OsuApiKeys = OsuApiKeys(name.getQualifiedName(), null)
// -------------------------------------------------------------------------
// Row5 type methods
// -------------------------------------------------------------------------
override fun fieldsRow(): Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?> = super.fieldsRow() as Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?>
/**
* Convenience mapping calling {@link SelectField#convertFrom(Function)}.
*/
fun <U> mapping(from: (Int?, String?, OffsetDateTime?, Boolean?, Boolean?) -> U): SelectField<U> = convertFrom(Records.mapping(from))
/**
* Convenience mapping calling {@link SelectField#convertFrom(Class,
* Function)}.
*/
fun <U> mapping(toType: Class<U>, from: (Int?, String?, OffsetDateTime?, Boolean?, Boolean?) -> U): SelectField<U> = convertFrom(toType, Records.mapping(from))
}

2
nise-backend/src/main/kotlin/com/nisemoe/generated/tables/UpdateUserQueue.kt
View File

@ -87,7 +87,7 @@ open class UpdateUserQueue(
/**
* The column <code>public.update_user_queue.processed_at</code>.
*/
val PROCESSED_AT: TableField<UpdateUserQueueRecord, OffsetDateTime?> = createField(DSL.name("processed_at"), SQLDataType.TIMESTAMPWITHTIMEZONE(6).defaultValue(DSL.field(DSL.raw("CURRENT_TIMESTAMP"), SQLDataType.TIMESTAMPWITHTIMEZONE)), this, "")
val PROCESSED_AT: TableField<UpdateUserQueueRecord, OffsetDateTime?> = createField(DSL.name("processed_at"), SQLDataType.TIMESTAMPWITHTIMEZONE(6), this, "")
/**
* The column <code>public.update_user_queue.progress_current</code>.

121
nise-backend/src/main/kotlin/com/nisemoe/generated/tables/records/OsuApiKeysRecord.kt Normal file
View File

@ -0,0 +1,121 @@
/*
* This file is generated by jOOQ.
*/
package com.nisemoe.generated.tables.records
import com.nisemoe.generated.tables.OsuApiKeys
import java.time.OffsetDateTime
import org.jooq.Field
import org.jooq.Record1
import org.jooq.Record5
import org.jooq.Row5
import org.jooq.impl.UpdatableRecordImpl
/**
* This class is generated by jOOQ.
*/
@Suppress("UNCHECKED_CAST")
open class OsuApiKeysRecord private constructor() : UpdatableRecordImpl<OsuApiKeysRecord>(OsuApiKeys.OSU_API_KEYS), Record5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?> {
open var id: Int?
set(value): Unit = set(0, value)
get(): Int? = get(0) as Int?
open var apiKey: String
set(value): Unit = set(1, value)
get(): String = get(1) as String
open var createdAt: OffsetDateTime?
set(value): Unit = set(2, value)
get(): OffsetDateTime? = get(2) as OffsetDateTime?
@Suppress("INAPPLICABLE_JVM_NAME")
@set:JvmName("setIsValid")
open var isValid: Boolean?
set(value): Unit = set(3, value)
get(): Boolean? = get(3) as Boolean?
@Suppress("INAPPLICABLE_JVM_NAME")
@set:JvmName("setIsActive")
open var isActive: Boolean?
set(value): Unit = set(4, value)
get(): Boolean? = get(4) as Boolean?
// -------------------------------------------------------------------------
// Primary key information
// -------------------------------------------------------------------------
override fun key(): Record1<Int?> = super.key() as Record1<Int?>
// -------------------------------------------------------------------------
// Record5 type implementation
// -------------------------------------------------------------------------
override fun fieldsRow(): Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?> = super.fieldsRow() as Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?>
override fun valuesRow(): Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?> = super.valuesRow() as Row5<Int?, String?, OffsetDateTime?, Boolean?, Boolean?>
override fun field1(): Field<Int?> = OsuApiKeys.OSU_API_KEYS.ID
override fun field2(): Field<String?> = OsuApiKeys.OSU_API_KEYS.API_KEY
override fun field3(): Field<OffsetDateTime?> = OsuApiKeys.OSU_API_KEYS.CREATED_AT
override fun field4(): Field<Boolean?> = OsuApiKeys.OSU_API_KEYS.IS_VALID
override fun field5(): Field<Boolean?> = OsuApiKeys.OSU_API_KEYS.IS_ACTIVE
override fun component1(): Int? = id
override fun component2(): String = apiKey
override fun component3(): OffsetDateTime? = createdAt
override fun component4(): Boolean? = isValid
override fun component5(): Boolean? = isActive
override fun value1(): Int? = id
override fun value2(): String = apiKey
override fun value3(): OffsetDateTime? = createdAt
override fun value4(): Boolean? = isValid
override fun value5(): Boolean? = isActive
override fun value1(value: Int?): OsuApiKeysRecord {
set(0, value)
return this
}
override fun value2(value: String?): OsuApiKeysRecord {
set(1, value)
return this
}
override fun value3(value: OffsetDateTime?): OsuApiKeysRecord {
set(2, value)
return this
}
override fun value4(value: Boolean?): OsuApiKeysRecord {
set(3, value)
return this
}
override fun value5(value: Boolean?): OsuApiKeysRecord {
set(4, value)
return this
}
override fun values(value1: Int?, value2: String?, value3: OffsetDateTime?, value4: Boolean?, value5: Boolean?): OsuApiKeysRecord {
this.value1(value1)
this.value2(value2)
this.value3(value3)
this.value4(value4)
this.value5(value5)
return this
}
/**
* Create a detached, initialised OsuApiKeysRecord
*/
constructor(id: Int? = null, apiKey: String, createdAt: OffsetDateTime? = null, isValid: Boolean? = null, isActive: Boolean? = null): this() {
this.id = id
this.apiKey = apiKey
this.createdAt = createdAt
this.isValid = isValid
this.isActive = isActive
resetChangedOnNotNull()
}
}

6
nise-backend/src/main/kotlin/com/nisemoe/generated/tables/references/Tables.kt
View File

@ -6,6 +6,7 @@ package com.nisemoe.generated.tables.references
import com.nisemoe.generated.tables.Beatmaps
import com.nisemoe.generated.tables.FlywaySchemaHistory
import com.nisemoe.generated.tables.OsuApiKeys
import com.nisemoe.generated.tables.RedditPost
import com.nisemoe.generated.tables.Scores
import com.nisemoe.generated.tables.ScoresJudgements
@ -25,6 +26,11 @@ val BEATMAPS: Beatmaps = Beatmaps.BEATMAPS
*/
val FLYWAY_SCHEMA_HISTORY: FlywaySchemaHistory = FlywaySchemaHistory.FLYWAY_SCHEMA_HISTORY
/**
* The table <code>public.osu_api_keys</code>.
*/
val OSU_API_KEYS: OsuApiKeys = OsuApiKeys.OSU_API_KEYS
/**
* The table <code>public.reddit_post</code>.
*/

105
nise-backend/src/main/kotlin/com/nisemoe/nise/osu/OsuApi.kt
View File

@ -1,25 +1,65 @@
package com.nisemoe.nise.osu
import com.nisemoe.generated.tables.references.OSU_API_KEYS
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.Json
import org.jooq.DSLContext
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.InitializingBean
import org.springframework.beans.factory.annotation.Value
import org.springframework.stereotype.Service
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.util.concurrent.atomic.AtomicInteger
class InvalidOsuApiKeyException() : Exception()
@Service
class OsuApi(
private val tokenService: TokenService
) {
private val tokenService: TokenService,
private val dslContext: DSLContext
): InitializingBean {
private val logger = LoggerFactory.getLogger(javaClass)
@Value("\${OSU_API_KEY}")
private lateinit var osuApiKey: String
private lateinit var apiKeysList: List<String>
private val currentKeyIndex = AtomicInteger(0)
override fun afterPropertiesSet() {
this.loadApiKeys()
}
fun loadApiKeys() {
currentKeyIndex.set(0)
val keys = dslContext.select(OSU_API_KEYS.API_KEY)
.from(OSU_API_KEYS)
.where(OSU_API_KEYS.IS_VALID.eq(true))
.and(OSU_API_KEYS.IS_ACTIVE.eq(true))
.fetchInto(String::class.java)
this.apiKeysList = keys
this.logger.info("Loaded ${keys.size} valid API keys")
}
fun getCurrentApiKey(): String {
val key = apiKeysList[currentKeyIndex.get() % apiKeysList.size]
currentKeyIndex.getAndIncrement()
return key
}
fun setKeyAsInvalid(apiKey: String) {
dslContext.update(OSU_API_KEYS)
.set(OSU_API_KEYS.IS_VALID, false)
.where(OSU_API_KEYS.API_KEY.eq(apiKey))
.execute()
this.logger.info("Marked API key $apiKey as invalid")
this.loadApiKeys()
}
@OptIn(ExperimentalSerializationApi::class)
private val serializer = Json { ignoreUnknownKeys = true; explicitNulls = false }
@ -57,29 +97,41 @@ class OsuApi(
return this.sendWithRetry(request)
}
/**
* Retrieves the replay data for a given score ID from the Osu API.
* Efficiently cycles through the API keys to avoid rate limiting.
* It's limited to 10 requests per minute according to @ https://github.com/ppy/osu-api/wiki#get-replay-data
*
* @param scoreId The ID of the score (best_id)
*/
fun getReplay(scoreId: Long): OsuApiModels.ReplayResponse? {
val queryParams = mapOf(
"k" to this.osuApiKey,
"s" to scoreId,
"m" to 0 // [osu!std]
)
val response = this.doRequest("https://osu.ppy.sh/api/get_replay?", queryParams)
if(response == null) {
this.logger.info("Error loading replay data")
return null
while (apiKeysList.isNotEmpty()) {
val apiKey = this.getCurrentApiKey()
val queryParams = mapOf(
"k" to apiKey,
"s" to scoreId.toString(),
"m" to "0" // [osu!std]
)
try {
val response = this.doRequest("https://osu.ppy.sh/api/get_replay?", queryParams)
if (response != null && response.statusCode() == 200) {
try {
return serializer.decodeFromString(OsuApiModels.ReplayResponse.serializer(), response.body())
} catch(exception: Exception) {
this.logger.error("Failed to parse successful response: ${response.body()}")
this.logger.error(exception.stackTraceToString())
break
}
}
} catch (e: InvalidOsuApiKeyException) {
this.logger.error("Invalid API key detected: $apiKey, trying next...")
this.setKeyAsInvalid(apiKey)
}
}
return if (response.statusCode() == 200) {
try {
serializer.decodeFromString(OsuApiModels.ReplayResponse.serializer(), response.body())
} catch(exception: Exception) {
this.logger.error(response.body())
this.logger.error(exception.stackTraceToString())
return null
}
} else {
null
}
this.logger.error("Failed to load replay data after cycling through all keys.")
return null
}
fun getTopBeatmapScores(beatmapId: Int): OsuApiModels.BeatmapScores? {
@ -213,6 +265,11 @@ class OsuApi(
this.logger.debug("Result: {}", response.statusCode())
this.logger.debug("")
// Handle osu!api v1 invalid api key errors in a specific way; mark the key as invalid so that
// it won't be used again and return null to indicate that the request failed.
if(response.statusCode() == 401 && response.body() == "{\"error\":\"Please provide a valid API key.\"}")
throw InvalidOsuApiKeyException()
when(response.statusCode()) {
401 -> {
// If the status code is 401, the access token has expired or something, refresh it.

8
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/GlobalCache.kt
View File

@ -12,6 +12,8 @@ import kotlinx.coroutines.runBlocking
import org.slf4j.LoggerFactory
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Service
import org.springframework.util.StopWatch
import kotlin.math.roundToInt
@Service
class GlobalCache(
@ -27,9 +29,12 @@ class GlobalCache(
var statistics: Statistics? = null
var rssFeed: RssFeed? = null
val stopwatch = StopWatch()
// 10 minutes to ms = 600000
@Scheduled(fixedDelay = 600000, initialDelay = 0)
fun updateCaches() {
this.stopwatch.start()
logger.info("Updating the cache!")
runBlocking {
@ -43,6 +48,9 @@ class GlobalCache(
similarReplays = similarReplaysDeferred.await()
suspiciousScores = suspiciousScoresDeferred.await()
}
this.stopwatch.stop()
logger.info("Cache updated in {} seconds", String.format("%.2f", stopwatch.totalTimeSeconds))
}
}

2
nise-backend/src/main/kotlin/com/nisemoe/nise/scheduler/ImportScores.kt
View File

@ -563,7 +563,7 @@ class ImportScores(
// It's limited to 10 requests per minute according to @ https://github.com/ppy/osu-api/wiki#get-replay-data
// So, we sleep for 6 seconds.
Thread.sleep(6000)
// Thread.sleep(6000)
// Calculate UR
val processedReplay: CircleguardService.ReplayResponse? = try {

8
nise-backend/src/main/resources/db/migration/V0.0.1.019__create_osu_api_keys.sql Normal file
View File

@ -0,0 +1,8 @@
create table "public".osu_api_keys
(
id serial primary key,
api_key text not null,
created_at timestamp with time zone not null default current_timestamp,
is_valid boolean not null default true,
is_active boolean not null default true
);

2
nise-frontend/src/app/view-user/view-user.component.html
View File

@ -56,7 +56,7 @@
<div class="progress">
<span class="btn-info">updating now!</span> <span style="margin-left: 4px">|</span> progress: {{ this.userInfo.queue_details.progressCurrent != null ? this.userInfo.queue_details.progressCurrent : "?" }}/{{ this.userInfo.queue_details.progressTotal != null ? this.userInfo.queue_details.progressTotal : "?" }}
<ng-container *ngIf="!this.userInfo.queue_details.progressTotal && !this.userInfo.queue_details.progressCurrent; else loading">
<span style="font-weight: bold">(in queue)</span>
<span style="font-weight: bold">(in queue, be patient)</span>
</ng-container>
<ng-template #loading>
<app-cute-loading></app-cute-loading>