diff --git a/.gitignore b/.gitignore index b1dff0d..741b44e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ bin/ .vscode/ ### Mac OS ### -.DS_Store \ No newline at end of file +.DS_Store +/chromedriver.log diff --git a/src/main/kotlin/net/xintanalabs/rssotto/AppConfig.kt b/src/main/kotlin/net/xintanalabs/rssotto/AppConfig.kt deleted file mode 100644 index b8cd02e..0000000 --- a/src/main/kotlin/net/xintanalabs/rssotto/AppConfig.kt +++ /dev/null @@ -1,56 +0,0 @@ -package net.xintanalabs.rssotto.config - -import org.openqa.selenium.chrome.ChromeDriver -import org.openqa.selenium.chrome.ChromeDriverService -import org.openqa.selenium.chrome.ChromeOptions -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.http.converter.json.KotlinSerializationJsonHttpMessageConverter -import org.springframework.web.client.RestTemplate -import kotlinx.serialization.json.Json -import java.io.File - -@Configuration -class AppConfig { - - @Bean - fun restTemplate(): RestTemplate { - return RestTemplate() - } - - @Bean - fun json(): Json { - return Json { - ignoreUnknownKeys = true - coerceInputValues = true - prettyPrint = true - } - } - - @Bean - fun additionalMessageConverters(json: Json): List { - return listOf(KotlinSerializationJsonHttpMessageConverter(json)) - } - - @Bean(destroyMethod = "quit") - fun chromeDriver(): ChromeDriver { - // Set log file to suppress console output (optional) - val logFile = File("chromedriver.log") - val service = ChromeDriverService.Builder() - .withLogFile(logFile) // Redirect logs to a file - .withSilent(true) // Suppress console output - .withVerbose(false) // Explicitly disable verbose logging - .build() - - // Optional: Set hideCommandPromptWindow for Windows - System.setProperty("webdriver.chrome.hideCommandPromptWindow", "true") - - val options = ChromeOptions().apply { - addArguments("--headless") - addArguments("--disable-gpu") - addArguments("--no-sandbox") - addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124") - } - return ChromeDriver(service, options) - } -} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfig.kt b/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfig.kt index 81083ff..2306448 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfig.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfig.kt @@ -1,11 +1,62 @@ -package net.xintanalabs.rssotto +package net.xintanalabs.rssotto.config -import kotlinx.serialization.Serializable -import org.springframework.boot.context.properties.ConfigurationProperties -import org.springframework.boot.context.properties.bind.ConstructorBinding +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.chrome.ChromeDriverService +import org.openqa.selenium.chrome.ChromeOptions +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.http.converter.json.KotlinSerializationJsonHttpMessageConverter +import org.springframework.web.client.RestTemplate +import kotlinx.serialization.json.Json +import org.springframework.web.reactive.function.client.WebClient +import java.io.File -@Serializable -@ConfigurationProperties(prefix = "version-checker") -data class ApplicationConfig( - val intervalMinutes: Int -) \ No newline at end of file +@Configuration +class ApplicationConfig { + + @Bean + fun restTemplate(): RestTemplate { + return RestTemplate() + } + + @Bean + fun json(): Json { + return Json { + ignoreUnknownKeys = true + coerceInputValues = true + prettyPrint = true + } + } + + @Bean + fun webClient(builder: WebClient.Builder): WebClient { + return builder.build() + } + + @Bean + fun additionalMessageConverters(json: Json): List { + return listOf(KotlinSerializationJsonHttpMessageConverter(json)) + } + + @Bean(destroyMethod = "quit") + fun chromeDriver(): ChromeDriver { + // Set log file to suppress console output (optional) + val logFile = File("chromedriver.log") + val service = ChromeDriverService.Builder() + .withLogFile(logFile) // Redirect logs to a file + .withSilent(true) // Suppress console output + .withVerbose(false) // Explicitly disable verbose logging + .build() + + // Optional: Set hideCommandPromptWindow for Windows + System.setProperty("webdriver.chrome.hideCommandPromptWindow", "true") + + val options = ChromeOptions().apply { + addArguments("--headless") + addArguments("--disable-gpu") + addArguments("--no-sandbox") + addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124") + } + return ChromeDriver(service, options) + } +} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfigProperties.kt b/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfigProperties.kt new file mode 100644 index 0000000..8b7a37a --- /dev/null +++ b/src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfigProperties.kt @@ -0,0 +1,10 @@ +package net.xintanalabs.rssotto + +import kotlinx.serialization.Serializable +import org.springframework.boot.context.properties.ConfigurationProperties + +@Serializable +@ConfigurationProperties(prefix = "version-checker") +data class ApplicationConfigProperties( + val intervalMinutes: Int +) \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/RssottoApplication.kt b/src/main/kotlin/net/xintanalabs/rssotto/RssottoApplication.kt index 38985cf..e6f9264 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/RssottoApplication.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/RssottoApplication.kt @@ -2,8 +2,10 @@ package net.xintanalabs.rssotto import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.runApplication +import org.springframework.scheduling.annotation.EnableScheduling @SpringBootApplication +@EnableScheduling class RssotoApplication fun main(args: Array) { diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/ApiChecker.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/ApiChecker.kt index 960e4c8..6cad913 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/ApiChecker.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/ApiChecker.kt @@ -9,8 +9,8 @@ import org.springframework.web.reactive.function.client.awaitExchange @Component class ApiChecker(private val webClient: WebClient) : IVersionChecker { - override suspend fun getLatestVersion(paramsDict: Map, source: Source): String { - val url = paramsDict["url"]?.takeIf { it.isNotEmpty() } + override suspend fun getLatestVersion(paramsDict: Map): String { + val url = paramsDict["url"]?.takeIf { it.isNotEmpty() } ?: throw IllegalArgumentException("API URL required") val jsonPath = paramsDict["jsonPath"]?.takeIf { it.isNotEmpty() } ?: throw IllegalArgumentException("jsonPath required") diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/IVersionChecker.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/IVersionChecker.kt index f359e14..4f688a5 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/IVersionChecker.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/IVersionChecker.kt @@ -3,5 +3,5 @@ package net.xintanalabs.rssotto.components.checkers import net.xintanalabs.rssotto.model.Source interface IVersionChecker { - suspend fun getLatestVersion(paramsDict: Map, source: Source): String + suspend fun getLatestVersion(paramsDict: Map): String } \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/VersionCheckTask.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/VersionCheckTask.kt deleted file mode 100644 index 81cf7c1..0000000 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/VersionCheckTask.kt +++ /dev/null @@ -1,70 +0,0 @@ -package net.xintanalabs.rssotto.components.checkers - -import kotlinx.coroutines.* -import net.xintanalabs.rssotto.ApplicationConfig -import net.xintanalabs.rssotto.model.App -import net.xintanalabs.rssotto.model.CheckerType -import net.xintanalabs.rssotto.model.Source -import net.xintanalabs.rssotto.services.AppService -import net.xintanalabs.rssotto.services.CheckerTypeService -import net.xintanalabs.rssotto.services.SourceService -import org.slf4j.LoggerFactory -import org.springframework.scheduling.annotation.Async -import org.springframework.scheduling.annotation.Scheduled -import org.springframework.stereotype.Component - -@Component -class VersionCheckTask( - private val config: ApplicationConfig, - private val appService: AppService, - private val sourceService: SourceService, - private val checkerTypeService: CheckerTypeService - //private val versionService: VersionService -) { - - private val logger = LoggerFactory.getLogger(VersionCheckTask::class.java) - private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - - //@Scheduled(fixedRateString = "\${version-checker.interval-minutes}000", initialDelay = 1000) - //@Async - fun checkVersions() { - scope.launch { - val apps: List = appService.getAllApps() - var sources: List = sourceService.getAllSources() - var checkerTypes: List = checkerTypeService.getAllCheckerTypes() - logger.info("Starting version check task for ${apps.size} sources") - apps.map { app -> - // async { - // try { - /* - val source: Source? = sources.find { s -> s.id == app.source } - if (source != null) { - val checkerType: CheckerType? = checkerTypes.find { ct -> ct.id == source.checkerType } - - - app.params - val params = mutableMapOf() - app.url.let { params["url"] = it } - app.jsonPath?.let { params["jsonPath"] = it } - app.regex?.let { params["regex"] = it } - app.mode?.let { params["mode"] = it } - - val sourceDef = SourceDef( - defaults = mapOf( - "regex" to (app.regex ?: ""), - "mode" to (app.mode ?: "default") - ) - ) - - val version = versionService.checkVersion(app.type, params, sourceDef) - logger.info("Version for ${app.url} (${app.type}): $version")*/ - // Optionally save to DB (if using JPA) - } - //} catch (e: Exception) { - //logger.error("Failed to check version for ${app.url}: ${e.message}", e) - //} - // } - // }.awaitAll() - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/WebConfig.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/WebConfig.kt deleted file mode 100644 index 526bdec..0000000 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/WebConfig.kt +++ /dev/null @@ -1,38 +0,0 @@ -package net.xintanalabs.rssotto.components.checkers - -import org.openqa.selenium.chrome.ChromeDriver -import org.openqa.selenium.chrome.ChromeDriverService -import org.openqa.selenium.chrome.ChromeOptions -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.springframework.web.client.RestTemplate -import org.springframework.web.reactive.function.client.WebClient -import java.io.File - -@Configuration -class WebConfig { - @Bean - fun restTemplate(): RestTemplate = RestTemplate() - - @Bean - fun webClient(): WebClient = WebClient.builder().build() - - @Bean - fun chromeDriver(): ChromeDriver { - val logFile = File("chromedriver.log") - val service = ChromeDriverService.Builder() - .withLogFile(logFile) // Redirect logs to a file - .withSilent(true) // Suppress console output - .withVerbose(false) // Explicitly disable verbose logging - .build() - - val options = ChromeOptions().apply { - addArguments("--headless") - addArguments("--disable-gpu") - addArguments("--no-sandbox") - addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/91.0.4472.124") - } - - return ChromeDriver(service, options) - } -} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt index 8293067..cb07869 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt @@ -3,7 +3,9 @@ package net.xintanalabs.rssotto.components.checkers.scrape import kotlinx.coroutines.delay import net.xintanalabs.rssotto.components.checkers.IVersionChecker import net.xintanalabs.rssotto.model.Source +import net.xintanalabs.rssotto.tasks.ScheduledTasks import org.openqa.selenium.chrome.ChromeDriver +import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import org.springframework.web.client.RestTemplate import java.util.regex.Pattern @@ -13,31 +15,29 @@ class ScrapeChecker( private val restTemplate: RestTemplate, private val chromeDriver: ChromeDriver ) : IVersionChecker { - override suspend fun getLatestVersion(paramsDict: Map, source: Source): String { + + private val log = LoggerFactory.getLogger(ScrapeChecker::class.java) + + override suspend fun getLatestVersion(paramsDict: Map): String { val url = paramsDict["url"]?.takeIf { it.isNotEmpty() } ?: throw IllegalArgumentException("URL required") - - val mode = getValueOrDefault(paramsDict, "mode", source) + log.info("Url : {}", url) + val mode = paramsDict["mode"] + log.info("Mode : {}", mode) val fetcher: IScrapeFetcher = when (mode) { "selenium" -> SeleniumFetcher(chromeDriver) "jsoup" -> JSoupFetcher() else -> DefaultScrapeFetcher(restTemplate) } val response = fetcher.fetch(url) - val cleanedResponse = response.replace(">\\s+<".toRegex(), "><") - val regex = getValueOrDefault(paramsDict, "regex", source) - + log.info("Response : {}", cleanedResponse) + val regex = paramsDict["regex"] + log.info("Regex : {}", regex) val match = Pattern.compile(regex).matcher(cleanedResponse) if (!match.find() || match.groupCount() < 1) { throw Exception("No match with regex in response") } return match.group(1) } - - private fun getValueOrDefault(dict: Map, key: String, source: Source): String { - return dict[key]?.takeIf { it.isNotEmpty() } - ?: source.defaults[key] - ?: "" - } } \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/controller/LatestVersionFinderController.kt b/src/main/kotlin/net/xintanalabs/rssotto/controller/LatestVersionFinderController.kt new file mode 100644 index 0000000..fea1301 --- /dev/null +++ b/src/main/kotlin/net/xintanalabs/rssotto/controller/LatestVersionFinderController.kt @@ -0,0 +1,33 @@ +package net.xintanalabs.rssotto.controller + +import net.xintanalabs.rssotto.constants.Constants +import net.xintanalabs.rssotto.services.AppService +import net.xintanalabs.rssotto.services.CheckerTypeService +import net.xintanalabs.rssotto.services.LatestVersionFinderService +import net.xintanalabs.rssotto.services.SourceService +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("${Constants.API_BASE_PATH_V0}/exec") +class LatestVersionFinderController( + private val latestVersionFinderService: LatestVersionFinderService, + private val appService: AppService, + private val sourceService: SourceService, + private val checkerTypeService: CheckerTypeService + ) { + @PostMapping("/version-check") + suspend fun executeVersionCheck(): String? { + return try { + val apps = appService.getAllApps() + val sources = sourceService.getAllSources() + val checkerTypes = checkerTypeService.getAllCheckerTypes() + latestVersionFinderService.getAllLatestAppVersions(apps,sources, checkerTypes) + + "ok" + } catch(e: Exception) { + e.message + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/controller/SourceController.kt b/src/main/kotlin/net/xintanalabs/rssotto/controller/SourceController.kt index c6eba90..f1b02e2 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/controller/SourceController.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/controller/SourceController.kt @@ -14,11 +14,19 @@ import org.springframework.web.bind.annotation.RestController class SourceController(private val sourceService: SourceService) { @PostMapping suspend fun createSource(@RequestBody source: Source): Source { - return sourceService.createSource(source) + return try { + sourceService.createSource(source) + } catch (e: Exception) { + throw RuntimeException(e.message, e) + } } @GetMapping suspend fun getAllSources(): List { - return sourceService.getAllSources() + return try { + sourceService.getAllSources() + } catch (e: Exception) { + throw RuntimeException(e.message, e) + } } } \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/model/Source.kt b/src/main/kotlin/net/xintanalabs/rssotto/model/Source.kt index eeca4ee..3c7d8fc 100644 --- a/src/main/kotlin/net/xintanalabs/rssotto/model/Source.kt +++ b/src/main/kotlin/net/xintanalabs/rssotto/model/Source.kt @@ -2,11 +2,13 @@ package net.xintanalabs.rssotto.model import com.fasterxml.jackson.annotation.JsonInclude import org.springframework.data.annotation.Id +import org.springframework.data.mongodb.core.mapping.Field @JsonInclude(JsonInclude.Include.NON_NULL) data class Source( @Id val id: String? = null, val name: String, + @Field("type") val checkerType: String, val defaults: Map = mapOf() ) diff --git a/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt b/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt new file mode 100644 index 0000000..c44e7ef --- /dev/null +++ b/src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt @@ -0,0 +1,72 @@ +package net.xintanalabs.rssotto.services + +import net.xintanalabs.rssotto.components.checkers.CheckerFactory +import net.xintanalabs.rssotto.components.checkers.IVersionChecker +import net.xintanalabs.rssotto.components.checkers.scrape.ScrapeChecker +import net.xintanalabs.rssotto.model.App +import net.xintanalabs.rssotto.model.CheckerType +import net.xintanalabs.rssotto.model.Source +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service + +@Service +class LatestVersionFinderService(private val checkerFactory: CheckerFactory) { + private val log = LoggerFactory.getLogger(LatestVersionFinderService::class.java) + + suspend fun getLatestAppVersion(app: App, sources: List, checkerTypes: List): String? { + val source: Source? = sources.find { s -> s.id == app.source } + var appVersion: String? = null + if (source != null) { + val checkerType: CheckerType? = checkerTypes.find { ct -> ct.id == source.checkerType } + + if (checkerType != null) { + val checker: IVersionChecker = checkerFactory.createChecker(checkerType.name) + val paramsMap: Map = mapOf( + "url" to getUrl(app, source), + "regex" to getRegex(app, source), + "mode" to getMode(app, source) + ) + + appVersion = checker.getLatestVersion(paramsMap) + log.info("App (${app.name}, latest versión ${appVersion}") + + } + } + return appVersion + } + + suspend fun getAllLatestAppVersions( + apps: List, + sources: List, + checkerTypes: List + ): Map { + val versionsMap: MutableMap = mutableMapOf() + apps.map { app -> + if (app.id !== null) { + val appLatestVersion: String? = getLatestAppVersion(app, sources, checkerTypes) + versionsMap[app.id] = appLatestVersion + } + } + return versionsMap + } + + private fun getValue(field: String, app: App, source: Source): String { + var value: String? = app.params[field] + if (value == null) { + value = source.defaults[field] + } + return value ?: "" + } + + private fun getMode(app: App, source: Source): String { + return getValue("mode", app, source) + } + + private fun getUrl(app: App, source: Source): String { + return getValue("url", app, source) + } + + private fun getRegex(app: App, source: Source): String { + return getValue("regex", app, source) + } +} \ No newline at end of file diff --git a/src/main/kotlin/net/xintanalabs/rssotto/tasks/ScheduledTasks.kt b/src/main/kotlin/net/xintanalabs/rssotto/tasks/ScheduledTasks.kt new file mode 100644 index 0000000..96fa197 --- /dev/null +++ b/src/main/kotlin/net/xintanalabs/rssotto/tasks/ScheduledTasks.kt @@ -0,0 +1,18 @@ +package net.xintanalabs.rssotto.tasks + +import org.slf4j.LoggerFactory +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Component +import java.text.SimpleDateFormat +import java.util.Date + +@Component +class ScheduledTasks { + private val log = LoggerFactory.getLogger(ScheduledTasks::class.java) + private val dateFormat = SimpleDateFormat("HH:mm:ss") + + @Scheduled(fixedRate = 500000) + fun reportCurrentTime() { + log.info("The time is now {}", dateFormat.format(Date())) + } +} \ No newline at end of file