arhuako/rssotto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working checker

Browse Source
This commit is contained in:
Jose Conde
2025-09-07 16:26:25 +02:00
parent 5f80c98ddd
commit 5f83d4aad4
15 changed files with 224 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -43,3 +43,4 @@ bin/
### Mac OS ### ### Mac OS ###
.DS_Store .DS_Store
/chromedriver.log

56
src/main/kotlin/net/xintanalabs/rssotto/AppConfig.kt
View File

@@ -1,56 +0,0 @@
package net.xintanalabs.rssotto.config
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.chrome.ChromeDriverService
import org.openqa.selenium.chrome.ChromeOptions
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.http.converter.json.KotlinSerializationJsonHttpMessageConverter
import org.springframework.web.client.RestTemplate
import kotlinx.serialization.json.Json
import java.io.File
@Configuration
class AppConfig {
@Bean
fun restTemplate(): RestTemplate {
return RestTemplate()
}
@Bean
fun json(): Json {
return Json {
ignoreUnknownKeys = true
coerceInputValues = true
prettyPrint = true
}
}
@Bean
fun additionalMessageConverters(json: Json): List<KotlinSerializationJsonHttpMessageConverter> {
return listOf(KotlinSerializationJsonHttpMessageConverter(json))
}
@Bean(destroyMethod = "quit")
fun chromeDriver(): ChromeDriver {
// Set log file to suppress console output (optional)
val logFile = File("chromedriver.log")
val service = ChromeDriverService.Builder()
.withLogFile(logFile) // Redirect logs to a file
.withSilent(true) // Suppress console output
.withVerbose(false) // Explicitly disable verbose logging
.build()
// Optional: Set hideCommandPromptWindow for Windows
System.setProperty("webdriver.chrome.hideCommandPromptWindow", "true")
val options = ChromeOptions().apply {
addArguments("--headless")
addArguments("--disable-gpu")
addArguments("--no-sandbox")
addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124")
}
return ChromeDriver(service, options)
}
}

69
src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfig.kt
View File

@@ -1,11 +1,62 @@
package net.xintanalabs.rssotto package net.xintanalabs.rssotto.config
import kotlinx.serialization.Serializable import org.openqa.selenium.chrome.ChromeDriver
import org.springframework.boot.context.properties.ConfigurationProperties import org.openqa.selenium.chrome.ChromeDriverService
import org.springframework.boot.context.properties.bind.ConstructorBinding import org.openqa.selenium.chrome.ChromeOptions
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.http.converter.json.KotlinSerializationJsonHttpMessageConverter
import org.springframework.web.client.RestTemplate
import kotlinx.serialization.json.Json
import org.springframework.web.reactive.function.client.WebClient
import java.io.File
@Serializable @Configuration
@ConfigurationProperties(prefix = "version-checker") class ApplicationConfig {
data class ApplicationConfig(
val intervalMinutes: Int @Bean
) fun restTemplate(): RestTemplate {
return RestTemplate()
}
@Bean
fun json(): Json {
return Json {
ignoreUnknownKeys = true
coerceInputValues = true
prettyPrint = true
}
}
@Bean
fun webClient(builder: WebClient.Builder): WebClient {
return builder.build()
}
@Bean
fun additionalMessageConverters(json: Json): List<KotlinSerializationJsonHttpMessageConverter> {
return listOf(KotlinSerializationJsonHttpMessageConverter(json))
}
@Bean(destroyMethod = "quit")
fun chromeDriver(): ChromeDriver {
// Set log file to suppress console output (optional)
val logFile = File("chromedriver.log")
val service = ChromeDriverService.Builder()
.withLogFile(logFile) // Redirect logs to a file
.withSilent(true) // Suppress console output
.withVerbose(false) // Explicitly disable verbose logging
.build()
// Optional: Set hideCommandPromptWindow for Windows
System.setProperty("webdriver.chrome.hideCommandPromptWindow", "true")
val options = ChromeOptions().apply {
addArguments("--headless")
addArguments("--disable-gpu")
addArguments("--no-sandbox")
addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124")
}
return ChromeDriver(service, options)
}
}

10
src/main/kotlin/net/xintanalabs/rssotto/ApplicationConfigProperties.kt Normal file
View File

@@ -0,0 +1,10 @@
package net.xintanalabs.rssotto
import kotlinx.serialization.Serializable
import org.springframework.boot.context.properties.ConfigurationProperties
@Serializable
@ConfigurationProperties(prefix = "version-checker")
data class ApplicationConfigProperties(
val intervalMinutes: Int
)

2
src/main/kotlin/net/xintanalabs/rssotto/RssottoApplication.kt
View File

@@ -2,8 +2,10 @@ package net.xintanalabs.rssotto
import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.boot.autoconfigure.SpringBootApplication
import org.springframework.boot.runApplication import org.springframework.boot.runApplication
import org.springframework.scheduling.annotation.EnableScheduling
@SpringBootApplication @SpringBootApplication
@EnableScheduling
class RssotoApplication class RssotoApplication
fun main(args: Array<String>) { fun main(args: Array<String>) {

4
src/main/kotlin/net/xintanalabs/rssotto/components/checkers/ApiChecker.kt
View File

@@ -9,8 +9,8 @@ import org.springframework.web.reactive.function.client.awaitExchange
@Component @Component
class ApiChecker(private val webClient: WebClient) : IVersionChecker { class ApiChecker(private val webClient: WebClient) : IVersionChecker {
override suspend fun getLatestVersion(paramsDict: Map<String, String>, source: Source): String { override suspend fun getLatestVersion(paramsDict: Map<String, String>): String {
val url = paramsDict["url"]?.takeIf { it.isNotEmpty() } val url = paramsDict["url"]?.takeIf { it.isNotEmpty() }
?: throw IllegalArgumentException("API URL required") ?: throw IllegalArgumentException("API URL required")
val jsonPath = paramsDict["jsonPath"]?.takeIf { it.isNotEmpty() } val jsonPath = paramsDict["jsonPath"]?.takeIf { it.isNotEmpty() }
?: throw IllegalArgumentException("jsonPath required") ?: throw IllegalArgumentException("jsonPath required")

2
src/main/kotlin/net/xintanalabs/rssotto/components/checkers/IVersionChecker.kt
View File

@@ -3,5 +3,5 @@ package net.xintanalabs.rssotto.components.checkers
import net.xintanalabs.rssotto.model.Source import net.xintanalabs.rssotto.model.Source
interface IVersionChecker { interface IVersionChecker {
suspend fun getLatestVersion(paramsDict: Map<String, String>, source: Source): String suspend fun getLatestVersion(paramsDict: Map<String, String>): String
} }

70
src/main/kotlin/net/xintanalabs/rssotto/components/checkers/VersionCheckTask.kt
View File

@@ -1,70 +0,0 @@
package net.xintanalabs.rssotto.components.checkers
import kotlinx.coroutines.*
import net.xintanalabs.rssotto.ApplicationConfig
import net.xintanalabs.rssotto.model.App
import net.xintanalabs.rssotto.model.CheckerType
import net.xintanalabs.rssotto.model.Source
import net.xintanalabs.rssotto.services.AppService
import net.xintanalabs.rssotto.services.CheckerTypeService
import net.xintanalabs.rssotto.services.SourceService
import org.slf4j.LoggerFactory
import org.springframework.scheduling.annotation.Async
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Component
@Component
class VersionCheckTask(
private val config: ApplicationConfig,
private val appService: AppService,
private val sourceService: SourceService,
private val checkerTypeService: CheckerTypeService
//private val versionService: VersionService
) {
private val logger = LoggerFactory.getLogger(VersionCheckTask::class.java)
private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob())
//@Scheduled(fixedRateString = "\${version-checker.interval-minutes}000", initialDelay = 1000)
//@Async
fun checkVersions() {
scope.launch {
val apps: List<App> = appService.getAllApps()
var sources: List<Source> = sourceService.getAllSources()
var checkerTypes: List<CheckerType> = checkerTypeService.getAllCheckerTypes()
logger.info("Starting version check task for ${apps.size} sources")
apps.map { app ->
// async {
// try {
/*
val source: Source? = sources.find { s -> s.id == app.source }
if (source != null) {
val checkerType: CheckerType? = checkerTypes.find { ct -> ct.id == source.checkerType }
app.params
val params = mutableMapOf<String, String>()
app.url.let { params["url"] = it }
app.jsonPath?.let { params["jsonPath"] = it }
app.regex?.let { params["regex"] = it }
app.mode?.let { params["mode"] = it }
val sourceDef = SourceDef(
defaults = mapOf(
"regex" to (app.regex ?: ""),
"mode" to (app.mode ?: "default")
)
)
val version = versionService.checkVersion(app.type, params, sourceDef)
logger.info("Version for ${app.url} (${app.type}): $version")*/
// Optionally save to DB (if using JPA)
}
//} catch (e: Exception) {
//logger.error("Failed to check version for ${app.url}: ${e.message}", e)
//}
// }
// }.awaitAll()
}
}
}

38
src/main/kotlin/net/xintanalabs/rssotto/components/checkers/WebConfig.kt
View File

@@ -1,38 +0,0 @@
package net.xintanalabs.rssotto.components.checkers
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.chrome.ChromeDriverService
import org.openqa.selenium.chrome.ChromeOptions
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.web.client.RestTemplate
import org.springframework.web.reactive.function.client.WebClient
import java.io.File
@Configuration
class WebConfig {
@Bean
fun restTemplate(): RestTemplate = RestTemplate()
@Bean
fun webClient(): WebClient = WebClient.builder().build()
@Bean
fun chromeDriver(): ChromeDriver {
val logFile = File("chromedriver.log")
val service = ChromeDriverService.Builder()
.withLogFile(logFile) // Redirect logs to a file
.withSilent(true) // Suppress console output
.withVerbose(false) // Explicitly disable verbose logging
.build()
val options = ChromeOptions().apply {
addArguments("--headless")
addArguments("--disable-gpu")
addArguments("--no-sandbox")
addArguments("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/91.0.4472.124")
}
return ChromeDriver(service, options)
}
}

24
src/main/kotlin/net/xintanalabs/rssotto/components/checkers/scrape/ScrapeChecker.kt
View File

@@ -3,7 +3,9 @@ package net.xintanalabs.rssotto.components.checkers.scrape
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
import net.xintanalabs.rssotto.components.checkers.IVersionChecker import net.xintanalabs.rssotto.components.checkers.IVersionChecker
import net.xintanalabs.rssotto.model.Source import net.xintanalabs.rssotto.model.Source
import net.xintanalabs.rssotto.tasks.ScheduledTasks
import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeDriver
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component import org.springframework.stereotype.Component
import org.springframework.web.client.RestTemplate import org.springframework.web.client.RestTemplate
import java.util.regex.Pattern import java.util.regex.Pattern
@@ -13,31 +15,29 @@ class ScrapeChecker(
private val restTemplate: RestTemplate, private val restTemplate: RestTemplate,
private val chromeDriver: ChromeDriver private val chromeDriver: ChromeDriver
) : IVersionChecker { ) : IVersionChecker {
override suspend fun getLatestVersion(paramsDict: Map<String, String>, source: Source): String {
private val log = LoggerFactory.getLogger(ScrapeChecker::class.java)
override suspend fun getLatestVersion(paramsDict: Map<String, String>): String {
val url = paramsDict["url"]?.takeIf { it.isNotEmpty() } val url = paramsDict["url"]?.takeIf { it.isNotEmpty() }
?: throw IllegalArgumentException("URL required") ?: throw IllegalArgumentException("URL required")
log.info("Url : {}", url)
val mode = getValueOrDefault(paramsDict, "mode", source) val mode = paramsDict["mode"]
log.info("Mode : {}", mode)
val fetcher: IScrapeFetcher = when (mode) { val fetcher: IScrapeFetcher = when (mode) {
"selenium" -> SeleniumFetcher(chromeDriver) "selenium" -> SeleniumFetcher(chromeDriver)
"jsoup" -> JSoupFetcher() "jsoup" -> JSoupFetcher()
else -> DefaultScrapeFetcher(restTemplate) else -> DefaultScrapeFetcher(restTemplate)
} }
val response = fetcher.fetch(url) val response = fetcher.fetch(url)
val cleanedResponse = response.replace(">\\s+<".toRegex(), "><") val cleanedResponse = response.replace(">\\s+<".toRegex(), "><")
val regex = getValueOrDefault(paramsDict, "regex", source) log.info("Response : {}", cleanedResponse)
val regex = paramsDict["regex"]
log.info("Regex : {}", regex)
val match = Pattern.compile(regex).matcher(cleanedResponse) val match = Pattern.compile(regex).matcher(cleanedResponse)
if (!match.find() || match.groupCount() < 1) { if (!match.find() || match.groupCount() < 1) {
throw Exception("No match with regex in response") throw Exception("No match with regex in response")
} }
return match.group(1) return match.group(1)
} }
private fun getValueOrDefault(dict: Map<String, String>, key: String, source: Source): String {
return dict[key]?.takeIf { it.isNotEmpty() }
?: source.defaults[key]
?: ""
}
} }

33
src/main/kotlin/net/xintanalabs/rssotto/controller/LatestVersionFinderController.kt Normal file
View File

@@ -0,0 +1,33 @@
package net.xintanalabs.rssotto.controller
import net.xintanalabs.rssotto.constants.Constants
import net.xintanalabs.rssotto.services.AppService
import net.xintanalabs.rssotto.services.CheckerTypeService
import net.xintanalabs.rssotto.services.LatestVersionFinderService
import net.xintanalabs.rssotto.services.SourceService
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("${Constants.API_BASE_PATH_V0}/exec")
class LatestVersionFinderController(
private val latestVersionFinderService: LatestVersionFinderService,
private val appService: AppService,
private val sourceService: SourceService,
private val checkerTypeService: CheckerTypeService
) {
@PostMapping("/version-check")
suspend fun executeVersionCheck(): String? {
return try {
val apps = appService.getAllApps()
val sources = sourceService.getAllSources()
val checkerTypes = checkerTypeService.getAllCheckerTypes()
latestVersionFinderService.getAllLatestAppVersions(apps,sources, checkerTypes)
"ok"
} catch(e: Exception) {
e.message
}
}
}

12
src/main/kotlin/net/xintanalabs/rssotto/controller/SourceController.kt
View File

@@ -14,11 +14,19 @@ import org.springframework.web.bind.annotation.RestController
class SourceController(private val sourceService: SourceService) { class SourceController(private val sourceService: SourceService) {
@PostMapping @PostMapping
suspend fun createSource(@RequestBody source: Source): Source { suspend fun createSource(@RequestBody source: Source): Source {
return sourceService.createSource(source) return try {
sourceService.createSource(source)
} catch (e: Exception) {
throw RuntimeException(e.message, e)
}
} }
@GetMapping @GetMapping
suspend fun getAllSources(): List<Source> { suspend fun getAllSources(): List<Source> {
return sourceService.getAllSources() return try {
sourceService.getAllSources()
} catch (e: Exception) {
throw RuntimeException(e.message, e)
}
} }
} }

2
src/main/kotlin/net/xintanalabs/rssotto/model/Source.kt
View File

@@ -2,11 +2,13 @@ package net.xintanalabs.rssotto.model
import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonInclude
import org.springframework.data.annotation.Id import org.springframework.data.annotation.Id
import org.springframework.data.mongodb.core.mapping.Field
@JsonInclude(JsonInclude.Include.NON_NULL) @JsonInclude(JsonInclude.Include.NON_NULL)
data class Source( data class Source(
@Id val id: String? = null, @Id val id: String? = null,
val name: String, val name: String,
@Field("type")
val checkerType: String, val checkerType: String,
val defaults: Map<String, String> = mapOf() val defaults: Map<String, String> = mapOf()
) )

72
src/main/kotlin/net/xintanalabs/rssotto/services/LatestVersionFinderService.kt Normal file
View File

@@ -0,0 +1,72 @@
package net.xintanalabs.rssotto.services
import net.xintanalabs.rssotto.components.checkers.CheckerFactory
import net.xintanalabs.rssotto.components.checkers.IVersionChecker
import net.xintanalabs.rssotto.components.checkers.scrape.ScrapeChecker
import net.xintanalabs.rssotto.model.App
import net.xintanalabs.rssotto.model.CheckerType
import net.xintanalabs.rssotto.model.Source
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
@Service
class LatestVersionFinderService(private val checkerFactory: CheckerFactory) {
private val log = LoggerFactory.getLogger(LatestVersionFinderService::class.java)
suspend fun getLatestAppVersion(app: App, sources: List<Source>, checkerTypes: List<CheckerType>): String? {
val source: Source? = sources.find { s -> s.id == app.source }
var appVersion: String? = null
if (source != null) {
val checkerType: CheckerType? = checkerTypes.find { ct -> ct.id == source.checkerType }
if (checkerType != null) {
val checker: IVersionChecker = checkerFactory.createChecker(checkerType.name)
val paramsMap: Map<String, String> = mapOf<String, String>(
"url" to getUrl(app, source),
"regex" to getRegex(app, source),
"mode" to getMode(app, source)
)
appVersion = checker.getLatestVersion(paramsMap)
log.info("App (${app.name}, latest versión ${appVersion}")
}
}
return appVersion
}
suspend fun getAllLatestAppVersions(
apps: List<App>,
sources: List<Source>,
checkerTypes: List<CheckerType>
): Map<String, String?> {
val versionsMap: MutableMap<String, String?> = mutableMapOf<String, String?>()
apps.map { app ->
if (app.id !== null) {
val appLatestVersion: String? = getLatestAppVersion(app, sources, checkerTypes)
versionsMap[app.id] = appLatestVersion
}
}
return versionsMap
}
private fun getValue(field: String, app: App, source: Source): String {
var value: String? = app.params[field]
if (value == null) {
value = source.defaults[field]
}
return value ?: ""
}
private fun getMode(app: App, source: Source): String {
return getValue("mode", app, source)
}
private fun getUrl(app: App, source: Source): String {
return getValue("url", app, source)
}
private fun getRegex(app: App, source: Source): String {
return getValue("regex", app, source)
}
}

18
src/main/kotlin/net/xintanalabs/rssotto/tasks/ScheduledTasks.kt Normal file
View File

@@ -0,0 +1,18 @@
package net.xintanalabs.rssotto.tasks
import org.slf4j.LoggerFactory
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.stereotype.Component
import java.text.SimpleDateFormat
import java.util.Date
@Component
class ScheduledTasks {
private val log = LoggerFactory.getLogger(ScheduledTasks::class.java)
private val dateFormat = SimpleDateFormat("HH:mm:ss")
@Scheduled(fixedRate = 500000)
fun reportCurrentTime() {
log.info("The time is now {}", dateFormat.format(Date()))
}
}