diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/JSoupFetcher.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/JSoupFetcher.kt index 3585d1d..c4731e2 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/JSoupFetcher.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/JSoupFetcher.kt @@ -1,16 +1,21 @@ package net.xintanalabs.rssotto.components.checkers.scrape +import kotlinx.coroutines.delay import org.jsoup.Jsoup import org.springframework.stereotype.Component @Component("jsoup") class JSoupFetcher: IScrapeFetcher { override suspend fun fetch(url: String): String { + delay(1000) // 1-second delay to avoid rate limiting return try { Jsoup.connect(url) .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") .header("Accept-Language", "en-US,en;q=0.5") + .header("Accept-Encoding", "gzip, deflate") + //.header("Connection", "keep-alive") + .header("Upgrade-Insecure-Requests", "1") .get() .html() } catch (e: Exception) { diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt index cb07869..008df8e 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt @@ -32,7 +32,7 @@ class ScrapeChecker( val response = fetcher.fetch(url) val cleanedResponse = response.replace(">\\s+<".toRegex(), "><") log.info("Response : {}", cleanedResponse) - val regex = paramsDict["regex"] + val regex: String = paramsDict["regex"] as String log.info("Regex : {}", regex) val match = Pattern.compile(regex).matcher(cleanedResponse) if (!match.find() || match.groupCount() < 1) { diff --git a/src/main/kotlin/net/xintanalabs/rssotto/model/App.kt b/src/main/kotlin/net/xintanalabs/rssotto/model/App.kt index 7f91ace..db58131 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/model/App.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/model/App.kt @@ -15,9 +15,10 @@ data class App( val fields: Map, val downloadUrl: String, val currentVersion: String, - val latestVersion: String, + val latestVersion: String? = null, val status: AppStatus, val createdAt: Long = 0, val updatedAt: Long = 0, - val lastCheckedAt: Long = 0 + val lastCheckedAt: Long = 0, + val active: Boolean = true ) diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/AppService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/AppService.kt index 28fbcab..116bc71 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/AppService.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/AppService.kt @@ -58,7 +58,7 @@ class AppService(private val mongoDBClient: MongoDBClient) { ) return mongoDBClient.updateOne( collection, - "uid", + "_id", id, updateFields, App::class.java @@ -68,4 +68,21 @@ class AppService(private val mongoDBClient: MongoDBClient) { } } + suspend fun updateLatestVersion(appId: String, latestVersion: String): Long { + try { + val updateFields = mapOf( + "latestVersion" to latestVersion + ) + return mongoDBClient.updateOne( + collection, + "_id", + appId, + updateFields, + App::class.java + ) + } catch (e: Exception) { + throw RuntimeException("Failed to update app: ${e.message}", e) + } + } + } \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/CheckerTypeService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/CheckerTypeService.kt index 0460e74..3992847 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/CheckerTypeService.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/CheckerTypeService.kt @@ -6,7 +6,7 @@ import net.xintanalabs.rssotto.model.CheckerType import org.springframework.stereotype.Service @Service -class CheckerTypeService (mongoDBClient: MongoDBClient) : ServiceBase(mongoDBClient){ +class CheckerTypeService (mongoDBClient: MongoDBClient) : ServiceMongoDBBase(mongoDBClient){ override val collection: String = Constants.COLLECTION_CHECKER_TYPES override val entityClass: Class = CheckerType::class.java diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt index c44e7ef..d6bbacb 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory import org.springframework.stereotype.Service @Service -class LatestVersionFinderService(private val checkerFactory: CheckerFactory) { +class LatestVersionFinderService(private val checkerFactory: CheckerFactory, private val versionService: VersionService) { private val log = LoggerFactory.getLogger(LatestVersionFinderService::class.java) suspend fun getLatestAppVersion(app: App, sources: List, checkerTypes: List): String? { @@ -19,7 +19,7 @@ class LatestVersionFinderService(private val checkerFactory: CheckerFactory) { if (source != null) { val checkerType: CheckerType? = checkerTypes.find { ct -> ct.id == source.checkerType } - if (checkerType != null) { + if (checkerType != null && app.id !=null) { val checker: IVersionChecker = checkerFactory.createChecker(checkerType.name) val paramsMap: Map = mapOf( "url" to getUrl(app, source), @@ -29,7 +29,7 @@ class LatestVersionFinderService(private val checkerFactory: CheckerFactory) { appVersion = checker.getLatestVersion(paramsMap) log.info("App (${app.name}, latest versión ${appVersion}") - + versionService.setVersion(app.id, appVersion) } } return appVersion @@ -42,7 +42,7 @@ class LatestVersionFinderService(private val checkerFactory: CheckerFactory) { ): Map { val versionsMap: MutableMap = mutableMapOf() apps.map { app -> - if (app.id !== null) { + if (app.id !== null && app.active) { val appLatestVersion: String? = getLatestAppVersion(app, sources, checkerTypes) versionsMap[app.id] = appLatestVersion } diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/ServiceBase.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/ServiceMongoDBBase.kt similarity index 89% rename from src/main/kotlin/net/xintanalabs/rssotto/services/ServiceBase.kt rename to src/main/kotlin/net/xintanalabs/rssotto/services/ServiceMongoDBBase.kt index 6e235b5..45072bb 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/ServiceBase.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/ServiceMongoDBBase.kt @@ -4,7 +4,7 @@ import kotlinx.coroutines.flow.toList import net.xintanalabs.rssotto.db.mongodb.MongoDBClient import org.springframework.beans.factory.annotation.Autowired -abstract class ServiceBase @Autowired constructor(protected val mongoDBClient: MongoDBClient) { +abstract class ServiceMongoDBBase @Autowired constructor(protected val mongoDBClient: MongoDBClient) { protected abstract val collection: String protected abstract val entityClass: Class protected val idField: String = "id" diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/SourceService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/SourceService.kt index 4551cfe..3409d1c 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/SourceService.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/SourceService.kt @@ -6,7 +6,7 @@ import net.xintanalabs.rssotto.model.Source import org.springframework.stereotype.Service @Service -class SourceService(mongoDBClient: MongoDBClient): ServiceBase(mongoDBClient) { +class SourceService(mongoDBClient: MongoDBClient): ServiceMongoDBBase(mongoDBClient) { override val collection: String = Constants.COLLECTION_SOURCES override val entityClass: Class = Source::class.java diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/TypeService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/TypeService.kt index 7c8a2c5..60933f0 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/services/TypeService.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/TypeService.kt @@ -1,13 +1,12 @@ package net.xintanalabs.rssotto.services -import kotlinx.coroutines.flow.toList import net.xintanalabs.rssotto.constants.Constants import net.xintanalabs.rssotto.db.mongodb.MongoDBClient import net.xintanalabs.rssotto.model.Type import org.springframework.stereotype.Service @Service -class TypeService(mongoDBClient: MongoDBClient) : ServiceBase(mongoDBClient) { +class TypeService(mongoDBClient: MongoDBClient) : ServiceMongoDBBase(mongoDBClient) { override val collection: String = Constants.COLLECTION_TYPES override val entityClass: Class = Type::class.java diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/VersionService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/VersionService.kt new file mode 100644 index 0000000..c5a5706 --- /dev/null +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/VersionService.kt @@ -0,0 +1,23 @@ +package net.xintanalabs.rssotto.services + +import net.xintanalabs.rssotto.db.mongodb.MongoDBClient +import org.springframework.stereotype.Service + + +@Service +class VersionService(private val appService: AppService) { + private val latestVersionsMap: MutableMap = mutableMapOf() + + suspend fun getVersion(appId: String): String? { + var version = latestVersionsMap[appId] + if (version == null) { + version = appService.getAppById(appId)?.latestVersion + } + return version + } + + suspend fun setVersion(appId: String, version: String) { + latestVersionsMap[appId] = version; + appService.updateLatestVersion(appId, version) + } +} \ No newline at end of file