From e8ab41fc425dc0002ab959d495c7b59cf49d3c53 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Thu, 6 Aug 2026 10:58:29 +0200 Subject: [PATCH 01/22] chore: prepare server 1.4.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 6d84d772..99db84dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.3.1 +appVersion=1.4.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000 From 022d15968b70367103bc0e83652b19852d0c638c Mon Sep 17 00:00:00 2001 From: Priveetee Date: Fri, 7 Aug 2026 22:28:37 +0200 Subject: [PATCH 02/22] fix: verify remote login capability --- openapi/components/instance.yaml | 2 +- .../YoutubeRemoteLoginReadinessService.kt | 20 +++++++++-- .../YoutubeRemoteLoginReadinessServiceTest.kt | 35 ++++++++++++++++--- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 676c77c6..385bb0fc 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -44,7 +44,7 @@ InstanceResponse: description: True only when the admin setting is enabled and remote login is ready. youtubeRemoteLoginReady: type: boolean - description: Non-secret readiness state for YouTube remote login. + description: True when Token accepts the shared secret and callback and its browser runtime is available. youtubeRemoteLoginUnavailableReason: type: string nullable: true diff --git a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt index 1e3ed387..5ee9abb3 100644 --- a/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt +++ b/src/main/kotlin/dev/typetype/server/services/YoutubeRemoteLoginReadinessService.kt @@ -3,8 +3,13 @@ package dev.typetype.server.services import dev.typetype.server.models.YoutubeRemoteLoginStatus import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody import java.util.concurrent.TimeUnit class YoutubeRemoteLoginReadinessService( @@ -13,6 +18,7 @@ class YoutubeRemoteLoginReadinessService( private val client: OkHttpClient = defaultClient(), private val nowMs: () -> Long = System::currentTimeMillis, ) { + private val json = Json { encodeDefaults = true } private var cachedStatus: YoutubeRemoteLoginStatus? = null private var cachedUntilMs: Long = 0 private val lock = Any() @@ -40,9 +46,14 @@ class YoutubeRemoteLoginReadinessService( } private fun probeToken(): YoutubeRemoteLoginStatus { + val internalToken = config.internalToken ?: return YoutubeRemoteLoginStatus.NotConfigured val request = Request.Builder() - .url("${config.serviceUrl.trimEnd('/')}/health") - .get() + .url("${config.serviceUrl.trimEnd('/')}/youtube-remote-login/readiness") + .header(INTERNAL_HEADER, internalToken) + .post( + json.encodeToString(YoutubeRemoteLoginReadinessRequest(config.callbackUrl)) + .toRequestBody(JSON_MEDIA_TYPE) + ) .build() return runCatching { client.newCall(request).execute().use { @@ -53,6 +64,8 @@ class YoutubeRemoteLoginReadinessService( private companion object { const val CACHE_TTL_MS = 30_000L + const val INTERNAL_HEADER = "X-Internal-Token" + val JSON_MEDIA_TYPE = "application/json".toMediaType() fun defaultClient(): OkHttpClient = OkHttpClient.Builder() @@ -62,3 +75,6 @@ class YoutubeRemoteLoginReadinessService( .build() } } + +@Serializable +private data class YoutubeRemoteLoginReadinessRequest(val callbackUrl: String) diff --git a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt b/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt index 7126a130..c66879f5 100644 --- a/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/YoutubeRemoteLoginReadinessServiceTest.kt @@ -11,6 +11,7 @@ import okhttp3.OkHttpClient import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody +import okio.Buffer import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -34,17 +35,22 @@ class YoutubeRemoteLoginReadinessServiceTest { } @Test - fun `token health failure returns token unreachable`() = runBlocking { - val service = service(config("secret"), sessionConfigured = true, client = client(500)) + fun `token without capability endpoint returns token unreachable`() = runBlocking { + val service = service(config("secret"), sessionConfigured = true, client = client(404)) assertEquals(YoutubeRemoteLoginStatus.TokenUnreachable, service.status(adminEnabled = true)) } @Test - fun `token health success returns ready`() = runBlocking { - val service = service(config("secret"), sessionConfigured = true, client = client(200)) + fun `token capability success returns ready`() = runBlocking { + val recorder = RequestRecorder() + val service = service(config("secret"), sessionConfigured = true, client = client(204, recorder = recorder)) assertEquals(YoutubeRemoteLoginStatus.Ready, service.status(adminEnabled = true)) + assertEquals("POST", recorder.method) + assertEquals("/youtube-remote-login/readiness", recorder.path) + assertEquals("secret", recorder.internalToken) + assertEquals("{\"callbackUrl\":\"http://server/internal/youtube-remote-login/callback\"}", recorder.body) } private fun service( @@ -61,9 +67,14 @@ class YoutubeRemoteLoginReadinessServiceTest { private fun config(internalToken: String?): YoutubeRemoteBrowserConfig = YoutubeRemoteBrowserConfig("http://token", "http://server", internalToken, 480_000, 2, 524_288, 4096, 2) - private fun client(code: Int, calls: Counter = Counter()): OkHttpClient = + private fun client( + code: Int, + calls: Counter = Counter(), + recorder: RequestRecorder? = null, + ): OkHttpClient = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> calls.value += 1 + recorder?.record(chain.request()) Response.Builder() .request(chain.request()) .protocol(Protocol.HTTP_1_1) @@ -76,4 +87,18 @@ class YoutubeRemoteLoginReadinessServiceTest { private class Counter { var value: Int = 0 } + + private class RequestRecorder { + var method: String? = null + var path: String? = null + var internalToken: String? = null + var body: String? = null + + fun record(request: okhttp3.Request) { + method = request.method + path = request.url.encodedPath + internalToken = request.header("X-Internal-Token") + body = Buffer().also { request.body?.writeTo(it) }.readUtf8() + } + } } From c8558cc7eabfa197226f8685d034e52c2f86f320 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:03 +0200 Subject: [PATCH 03/22] fix: restrict public proxy destinations --- openapi/paths/proxy.yaml | 6 +- .../server/services/BilibiliRangeProxy.kt | 8 +- .../server/services/HlsManifestService.kt | 5 +- .../server/services/NicoVideoProxyService.kt | 12 +- .../server/services/OkHttpProxyService.kt | 9 +- .../server/services/ProxyHttpExecutor.kt | 89 ++++++++++++ .../typetype/server/services/UrlValidator.kt | 132 ++++++++++++++---- 7 files changed, 215 insertions(+), 46 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt diff --git a/openapi/paths/proxy.yaml b/openapi/paths/proxy.yaml index b1162a6a..2c7685cd 100644 --- a/openapi/paths/proxy.yaml +++ b/openapi/paths/proxy.yaml @@ -3,8 +3,10 @@ Proxy: tags: [extraction] summary: Retrieve proxied media description: >- - Streams supported remote content. Existing clients that submit a YouTube timed-text URL are - routed through the dedicated subtitle resolver for compatibility; new clients should use + Streams media from the supported YouTube, NicoNico, and BiliBili delivery hosts. Other + destinations, non-public addresses, cross-provider redirects, and non-HTTPS URLs are + rejected. Existing clients that submit a YouTube timed-text URL are routed through the + dedicated subtitle resolver for compatibility; new clients should use /subtitles/youtube/{videoId}. parameters: - name: url diff --git a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt index 35736ea9..3268ea8e 100644 --- a/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt +++ b/src/main/kotlin/dev/typetype/server/services/BilibiliRangeProxy.kt @@ -2,7 +2,6 @@ package dev.typetype.server.services import dev.typetype.server.models.ExtractionResult import dev.typetype.server.models.ProxyResponse -import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import java.io.ByteArrayInputStream @@ -10,10 +9,13 @@ import java.io.IOException private const val BILIBILI_RANGE_ATTEMPTS = 3 -internal fun readBilibiliRangeWithRetry(client: OkHttpClient, request: Request): ExtractionResult { +internal fun readBilibiliRangeWithRetry( + execute: (Request) -> Response, + request: Request, +): ExtractionResult { var lastMessage = "Proxy fetch failed" for (attempt in 1..BILIBILI_RANGE_ATTEMPTS) { - runCatching { client.newCall(request).execute() } + runCatching { execute(request) } .onSuccess { response -> response.use { val result = it.readBilibiliRangeBytes() diff --git a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt index 15839531..cf12d6a9 100644 --- a/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt +++ b/src/main/kotlin/dev/typetype/server/services/HlsManifestService.kt @@ -13,11 +13,12 @@ import java.util.concurrent.ConcurrentHashMap class HlsManifestService( private val streamService: StreamService, - private val httpClient: OkHttpClient, + httpClient: OkHttpClient, cache: CacheService? = null, private val signManifestUrl: ((String) -> String)? = null, private val attestedYoutubeHls: suspend (String) -> String? = { null }, ) { + private val proxyHttp = ProxyHttpExecutor(httpClient) private val manifestCache = cache?.let(::HlsManifestCache) private val inFlight = ConcurrentHashMap>>() @@ -101,7 +102,7 @@ class HlsManifestService( .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) .apply { if (domandBid != null) header("Cookie", "domand_bid=$domandBid") } .build() - httpClient.newCall(request).execute() + proxyHttp.execute(request) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt index 6131c3ac..9c475dd1 100644 --- a/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/NicoVideoProxyService.kt @@ -40,13 +40,15 @@ internal fun rewriteNicoManifest(manifest: String, baseUrl: String, domandBid: S } } -class NicoVideoProxyService { +class NicoVideoProxyService(client: OkHttpClient = defaultNicoProxyClient()) { + private val executor = ProxyHttpExecutor(client) - private val client = OkHttpClient.Builder() + companion object { + private fun defaultNicoProxyClient() = OkHttpClient.Builder() .connectTimeout(10, TimeUnit.SECONDS) .readTimeout(30, TimeUnit.SECONDS) - .followRedirects(true) .build() + } suspend fun fetchManifest(rawUrl: String, domandBid: String? = null): ExtractionResult = withContext(Dispatchers.IO) { @@ -60,7 +62,7 @@ class NicoVideoProxyService { .url(manifestUrl) .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) if (resolvedBid != null) builder.header("Cookie", "domand_bid=$resolvedBid") - client.newCall(builder.build()).execute() + executor.execute(builder.build()) }.fold( onSuccess = { response -> val body = response.body @@ -95,7 +97,7 @@ class NicoVideoProxyService { .header("User-Agent", OkHttpProxyService.BROWSER_USER_AGENT) if (rangeHeader != null) builder.header("Range", rangeHeader) if (domandBid != null) builder.header("Cookie", "domand_bid=$domandBid") - client.newCall(builder.build()).execute() + executor.execute(builder.build()) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt index 27b4f1eb..ebde6bc6 100644 --- a/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt +++ b/src/main/kotlin/dev/typetype/server/services/OkHttpProxyService.kt @@ -21,7 +21,8 @@ internal fun rewriteHlsManifest(manifest: String): String = "/proxy?url=" + URLEncoder.encode(match.value, StandardCharsets.UTF_8) } -class OkHttpProxyService(private val client: OkHttpClient) : ProxyService { +class OkHttpProxyService(client: OkHttpClient) : ProxyService { + private val executor = ProxyHttpExecutor(client) override suspend fun pipe(url: String, rangeHeader: String?, domandBid: String?): ExtractionResult = withContext(Dispatchers.IO) { @@ -44,8 +45,10 @@ class OkHttpProxyService(private val client: OkHttpClient) : ProxyService { if (resolvedDomandBid != null && isNicoNico(cleanUrl)) builder.header("Cookie", "domand_bid=$resolvedDomandBid") if (rangeHeader != null) builder.header("Range", rangeHeader) val request = builder.build() - if (bilibili && rangeHeader != null) return@withContext readBilibiliRangeWithRetry(client, request) - client.newCall(request).execute() + if (bilibili && rangeHeader != null) { + return@withContext readBilibiliRangeWithRetry(executor::execute, request) + } + executor.execute(request) }.fold( onSuccess = { response -> val body = response.body diff --git a/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt b/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt new file mode 100644 index 00000000..4f27c759 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/ProxyHttpExecutor.kt @@ -0,0 +1,89 @@ +package dev.typetype.server.services + +import okhttp3.Dns +import okhttp3.HttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import java.io.IOException +import java.net.InetAddress +import java.net.InetSocketAddress +import java.net.Proxy +import java.net.UnknownHostException +import java.util.concurrent.ConcurrentHashMap + +internal class ProxyHttpExecutor( + client: OkHttpClient, + private val maxRedirects: Int = 5, +) { + private val dns = ValidatingProxyDns(client.dns) + private val client = client.newBuilder() + .dns(dns) + .followRedirects(false) + .followSslRedirects(false) + .build() + + fun execute(initialRequest: Request): Response { + var request = initialRequest + val initialTarget = requireProxyTarget(request.url.toString()) + repeat(maxRedirects + 1) { redirectCount -> + val target = requireProxyTarget(request.url.toString()) + if (target.provider != initialTarget.provider) { + throw ProxyTargetRejectedException("Cross-provider redirect is not allowed") + } + trustConfiguredProxy(target.url) + dns.lookup(target.url.host) + val response = client.newCall(request).execute() + val location = response.header("Location") + if (!response.isRedirect || location == null) return response + if (redirectCount == maxRedirects) { + response.close() + throw IOException("Too many proxy redirects") + } + val nextUrl = response.request.url.resolve(location) + response.close() + if (nextUrl == null) throw ProxyTargetRejectedException("Invalid proxy redirect") + request = request.newBuilder().url(nextUrl).build() + } + throw IOException("Too many proxy redirects") + } + + private fun trustConfiguredProxy(target: HttpUrl) { + val configured = client.proxy?.let(::listOf) + ?: client.proxySelector.select(target.toUri()) + configured.forEach { proxy -> + if (proxy.type() == Proxy.Type.DIRECT) return@forEach + val address = proxy.address() as? InetSocketAddress ?: return@forEach + dns.trustTransportHost(address.hostString) + } + } +} + +internal class ValidatingProxyDns(private val delegate: Dns) : Dns { + private val trustedTransportHosts = ConcurrentHashMap.newKeySet() + + override fun lookup(hostname: String): List { + val normalized = hostname.lowercase().trimEnd('.') + val trustedTransport = normalized in trustedTransportHosts + if (!trustedTransport && providerForProxyHost(normalized) == null) { + throw UnknownHostException("Unsupported proxy host") + } + val addresses = try { + delegate.lookup(hostname) + } catch (error: UnknownHostException) { + throw error + } catch (error: Exception) { + throw UnknownHostException(error.message ?: "Unable to resolve proxy host") + } + if (addresses.isEmpty()) throw UnknownHostException("Unable to resolve proxy host") + if (trustedTransport) return addresses + if (addresses.any { !isPublicProxyAddress(it) }) { + throw UnknownHostException("Blocked non-public proxy address") + } + return addresses + } + + fun trustTransportHost(hostname: String) { + trustedTransportHosts += hostname.lowercase().trimEnd('.') + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt b/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt index 8785e81a..d7b488cc 100644 --- a/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt +++ b/src/main/kotlin/dev/typetype/server/services/UrlValidator.kt @@ -1,43 +1,113 @@ package dev.typetype.server.services +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.net.InetAddress import java.net.URI -private val BLOCKED_HOST_SUFFIXES = listOf(".local", ".internal", ".localhost") -private val PRIVATE_RANGES = listOf( - intArrayOf(10, 0, 0, 0) to 8, - intArrayOf(172, 16, 0, 0) to 12, - intArrayOf(192, 168, 0, 0) to 16, - intArrayOf(127, 0, 0, 0) to 8, - intArrayOf(169, 254, 0, 0) to 16, - intArrayOf(0, 0, 0, 0) to 8, +internal enum class ProxyProvider { + YOUTUBE, + BILIBILI, + NICONICO, +} + +internal data class ProxyTarget( + val url: HttpUrl, + val provider: ProxyProvider, +) + +internal class ProxyTargetRejectedException(message: String) : IllegalArgumentException(message) + +private val BLOCKED_IPV4_RANGES = listOf( + ipv4(0, 0, 0, 0) to 8, + ipv4(10, 0, 0, 0) to 8, + ipv4(100, 64, 0, 0) to 10, + ipv4(127, 0, 0, 0) to 8, + ipv4(169, 254, 0, 0) to 16, + ipv4(172, 16, 0, 0) to 12, + ipv4(192, 0, 0, 0) to 24, + ipv4(192, 0, 2, 0) to 24, + ipv4(192, 168, 0, 0) to 16, + ipv4(198, 18, 0, 0) to 15, + ipv4(198, 51, 100, 0) to 24, + ipv4(203, 0, 113, 0) to 24, + ipv4(224, 0, 0, 0) to 4, + ipv4(240, 0, 0, 0) to 4, ) -internal fun validateProxyUrl(raw: String): String? { - val uri = runCatching { URI(raw) }.getOrElse { return "Malformed URL" } - val scheme = uri.scheme?.lowercase() ?: return "Missing URL scheme" - if (scheme != "http" && scheme != "https") return "Unsupported URL scheme: $scheme" - val host = uri.host?.lowercase() ?: return "Missing URL host" - if (host == "localhost") return "Blocked host" - if (BLOCKED_HOST_SUFFIXES.any { host.endsWith(it) }) return "Blocked host" - val addr = runCatching { InetAddress.getByName(host) }.getOrElse { return null } - val bytes = addr.address - if (bytes.size != 4) return null - val octets = bytes.map { it.toInt() and 0xFF } - for ((prefix, bits) in PRIVATE_RANGES) { - if (isInRange(octets, prefix, bits)) return "Blocked private address" +private val BLOCKED_IPV6_RANGES = listOf( + byteArrayOf(0x20, 0x01, 0x00, 0x00) to 32, + byteArrayOf(0x20, 0x01, 0x00, 0x02, 0x00, 0x00) to 48, + byteArrayOf(0x20, 0x01, 0x00, 0x10) to 28, + byteArrayOf(0x20, 0x01, 0x00, 0x20) to 28, + byteArrayOf(0x20, 0x01, 0x0D, 0xB8.toByte()) to 32, + byteArrayOf(0x20, 0x02) to 16, +) + +internal fun validateProxyUrl(raw: String): String? = + runCatching { requireProxyTarget(raw) } + .exceptionOrNull() + ?.message + +internal fun requireProxyTarget(raw: String): ProxyTarget { + val uri = runCatching { URI(raw) }.getOrElse { throw ProxyTargetRejectedException("Malformed URL") } + val scheme = uri.scheme?.lowercase() ?: throw ProxyTargetRejectedException("Missing URL scheme") + if (scheme != "https") throw ProxyTargetRejectedException("Unsupported URL scheme: $scheme") + if (uri.rawUserInfo != null) throw ProxyTargetRejectedException("URL credentials are not allowed") + if (uri.host == null) throw ProxyTargetRejectedException("Missing URL host") + val url = raw.toHttpUrlOrNull() ?: throw ProxyTargetRejectedException("Malformed URL") + if (url.username.isNotEmpty() || url.password.isNotEmpty()) { + throw ProxyTargetRejectedException("URL credentials are not allowed") } - return null + if (url.port != 443) throw ProxyTargetRejectedException("Unsupported proxy port") + val provider = providerForProxyHost(url.host) + ?: throw ProxyTargetRejectedException("Unsupported proxy host") + return ProxyTarget(url, provider) } -private fun isInRange(octets: List, prefix: IntArray, bits: Int): Boolean { - var remaining = bits - for (i in prefix.indices) { - val maskBits = remaining.coerceIn(0, 8) - val mask = if (maskBits == 0) 0 else (0xFF shl (8 - maskBits)) and 0xFF - if ((octets[i] and mask) != (prefix[i] and mask)) return false - remaining -= maskBits - if (remaining <= 0) break +internal fun providerForProxyHost(rawHost: String): ProxyProvider? { + val host = rawHost.lowercase().trimEnd('.') + return when { + host.matchesHost("googlevideo.com") || + host.matchesHost("ytimg.com") || + host.matchesHost("ggpht.com") || + host == "yt3.googleusercontent.com" -> ProxyProvider.YOUTUBE + host.matchesHost("bilivideo.com") || + host.matchesHost("bilivideo.cn") || + host.matchesHost("hdslb.com") || + host == "upos-hz-mirrorakam.akamaized.net" -> ProxyProvider.BILIBILI + host.matchesHost("nicovideo.jp") || host.matchesHost("nimg.jp") -> ProxyProvider.NICONICO + else -> null } - return true +} + +internal fun isPublicProxyAddress(address: InetAddress): Boolean { + if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || + address.isSiteLocalAddress || address.isMulticastAddress + ) return false + val bytes = address.address + return when (bytes.size) { + 4 -> BLOCKED_IPV4_RANGES.none { (prefix, bits) -> hasPrefix(bytes, prefix, bits) } + 16 -> isPublicIpv6(bytes) + else -> false + } +} + +private fun isPublicIpv6(bytes: ByteArray): Boolean { + if ((bytes[0].toInt() and 0xE0) != 0x20) return false + return BLOCKED_IPV6_RANGES.none { (prefix, bits) -> hasPrefix(bytes, prefix, bits) } +} + +private fun String.matchesHost(suffix: String): Boolean = this == suffix || endsWith(".$suffix") + +private fun ipv4(a: Int, b: Int, c: Int, d: Int): ByteArray = + byteArrayOf(a.toByte(), b.toByte(), c.toByte(), d.toByte()) + +private fun hasPrefix(address: ByteArray, prefix: ByteArray, bits: Int): Boolean { + val fullBytes = bits / 8 + for (index in 0 until fullBytes) if (address[index] != prefix[index]) return false + val remaining = bits % 8 + if (remaining == 0) return true + val mask = 0xFF shl (8 - remaining) + return (address[fullBytes].toInt() and mask) == (prefix[fullBytes].toInt() and mask) } From ba098b8686e0bb8ca50dfebd4d98f9e9cf7e5d29 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:19 +0200 Subject: [PATCH 04/22] test: cover proxy destination validation --- .../typetype/server/BilibiliRangeProxyTest.kt | 8 +- .../server/HlsManifestServiceCacheTest.kt | 31 ++++--- .../dev/typetype/server/UrlValidatorTest.kt | 81 ++++++++++++++++--- 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt index 69b1a63f..796c59cc 100644 --- a/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt +++ b/src/test/kotlin/dev/typetype/server/BilibiliRangeProxyTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.ExtractionResult import dev.typetype.server.services.OkHttpProxyService import kotlinx.coroutines.runBlocking import okhttp3.OkHttpClient +import okhttp3.Dns import okhttp3.Protocol import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody @@ -11,6 +12,7 @@ import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import java.io.IOException +import java.net.InetAddress class BilibiliRangeProxyTest { @@ -18,7 +20,9 @@ class BilibiliRangeProxyTest { fun `BiliBili range proxy retries transport failures`() = runBlocking { var calls = 0 val bytes = byteArrayOf(1, 2, 3, 4) - val client = OkHttpClient.Builder().addInterceptor { chain -> + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor { chain -> calls += 1 val request = chain.request() assertEquals(OkHttpProxyService.BILIBILI_USER_AGENT, request.header("User-Agent")) @@ -36,7 +40,7 @@ class BilibiliRangeProxyTest { .header("Content-Range", "bytes 0-3/4") .body(bytes.toResponseBody()) .build() - }.build() + }.build() val service = OkHttpProxyService(client) val result = service.pipe( diff --git a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt index 54b815c6..ada3a2a6 100644 --- a/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt +++ b/src/test/kotlin/dev/typetype/server/HlsManifestServiceCacheTest.kt @@ -7,6 +7,7 @@ import dev.typetype.server.services.HlsManifestService import dev.typetype.server.services.StreamService import kotlinx.coroutines.test.runTest import okhttp3.Interceptor +import okhttp3.Dns import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient import okhttp3.Protocol @@ -15,12 +16,13 @@ import okhttp3.ResponseBody.Companion.toResponseBody import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.net.InetAddress class HlsManifestServiceCacheTest { @Test fun `hls manifests are cached briefly by manifest url`() = runTest { var calls = 0 - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> calls += 1 Response.Builder() .request(chain.request()) @@ -29,9 +31,9 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U\nsegment.ts".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val service = HlsManifestService(NoopStreamService, client, InMemoryCache()) - val url = "https://example.com/master.m3u8" + val url = "https://manifest.googlevideo.com/master.m3u8" service.hlsManifest(url) service.hlsManifest(url) @@ -43,7 +45,7 @@ class HlsManifestServiceCacheTest { fun `attested manifest is scoped to youtube live`() = runTest { val requestedUrls = mutableListOf() val attestedVideoIds = mutableListOf() - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> requestedUrls += chain.request().url.toString() Response.Builder() .request(chain.request()) @@ -52,13 +54,13 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val streams = FixedStreamService( - testStreamResponse(hlsUrl = "https://example.com/legacy.m3u8").copy(isLive = true), + testStreamResponse(hlsUrl = "https://upos-hz-mirrorakam.akamaized.net/legacy.m3u8").copy(isLive = true), ) val service = HlsManifestService(streams, client, attestedYoutubeHls = { videoId -> attestedVideoIds += videoId - "https://example.com/attested.m3u8" + "https://manifest.googlevideo.com/attested.m3u8" }) val publicResult = service.hlsManifest("https://youtube.com/watch?v=test-id") @@ -73,9 +75,9 @@ class HlsManifestServiceCacheTest { assertEquals(listOf("test-id", "session-id"), attestedVideoIds) assertEquals( listOf( - "https://example.com/attested.m3u8", - "https://example.com/attested.m3u8", - "https://example.com/legacy.m3u8", + "https://manifest.googlevideo.com/attested.m3u8", + "https://manifest.googlevideo.com/attested.m3u8", + "https://upos-hz-mirrorakam.akamaized.net/legacy.m3u8", ), requestedUrls, ) @@ -84,7 +86,7 @@ class HlsManifestServiceCacheTest { @Test fun `NicoNico manifests use signed cookie and proxy segments`() = runTest { val requests = mutableListOf>() - val client = OkHttpClient.Builder().addInterceptor(Interceptor { chain -> + val client = proxyTestClient(Interceptor { chain -> requests += chain.request().url.toString() to chain.request().header("Cookie") Response.Builder() .request(chain.request()) @@ -93,7 +95,7 @@ class HlsManifestServiceCacheTest { .message("OK") .body("#EXTM3U\nsegment.cmfa".toResponseBody("application/vnd.apple.mpegurl".toMediaType())) .build() - }).build() + }) val service = HlsManifestService(NoopStreamService, client) val result = service.hlsManifest( @@ -110,6 +112,11 @@ class HlsManifestServiceCacheTest { } } +private fun proxyTestClient(interceptor: Interceptor): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(interceptor) + .build() + private object NoopStreamService : StreamService { override suspend fun getStreamInfo(url: String): ExtractionResult = ExtractionResult.Failure("unused") diff --git a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt b/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt index a0138de9..fc3d0252 100644 --- a/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt +++ b/src/test/kotlin/dev/typetype/server/UrlValidatorTest.kt @@ -1,32 +1,89 @@ package dev.typetype.server +import dev.typetype.server.services.ProxyProvider +import dev.typetype.server.services.isPublicProxyAddress +import dev.typetype.server.services.requireProxyTarget import dev.typetype.server.services.validateProxyUrl import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import java.net.InetAddress class UrlValidatorTest { @Test fun `rejects malformed and unsupported urls`() { assertEquals("Malformed URL", validateProxyUrl("not a url")) - assertEquals("Missing URL scheme", validateProxyUrl("example.com/video")) - assertEquals("Unsupported URL scheme: ftp", validateProxyUrl("ftp://example.com/video")) + assertEquals("Missing URL scheme", validateProxyUrl("i.ytimg.com/video")) + assertEquals("Unsupported URL scheme: http", validateProxyUrl("http://i.ytimg.com/video")) + assertEquals("Unsupported URL scheme: ftp", validateProxyUrl("ftp://i.ytimg.com/video")) assertEquals("Missing URL host", validateProxyUrl("https:///video")) + assertEquals("URL credentials are not allowed", validateProxyUrl("https://user@i.ytimg.com/video")) + assertEquals("Unsupported proxy port", validateProxyUrl("https://i.ytimg.com:8443/video")) } @Test - fun `blocks localhost and private ipv4 ranges`() { - assertEquals("Blocked host", validateProxyUrl("http://localhost/video")) - assertEquals("Blocked host", validateProxyUrl("https://demo.localhost/video")) - assertEquals("Blocked private address", validateProxyUrl("http://127.0.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://10.0.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://172.16.0.1/video")) - assertEquals("Blocked private address", validateProxyUrl("http://192.168.1.1/video")) + fun `allows only supported provider hosts`() { + assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://i.ytimg.com/image.jpg").provider) + assertEquals(ProxyProvider.YOUTUBE, requireProxyTarget("https://yt3.googleusercontent.com/avatar").provider) + assertEquals(ProxyProvider.BILIBILI, requireProxyTarget("https://i2.hdslb.com/image.jpg").provider) + assertEquals( + ProxyProvider.BILIBILI, + requireProxyTarget("https://upos-hz-mirrorakam.akamaized.net/video.m4s").provider, + ) + assertEquals( + ProxyProvider.NICONICO, + requireProxyTarget("https://delivery.domand.nicovideo.jp/video.m3u8").provider, + ) + assertNull(validateProxyUrl("https://r1---sn-a5mekn6z.googlevideo.com/videoplayback")) } @Test - fun `allows public and ipv6 addresses`() { - assertNull(validateProxyUrl("https://1.1.1.1/videoplayback?id=1")) - assertNull(validateProxyUrl("https://[2606:4700:4700::1111]/videoplayback?id=1")) + fun `rejects arbitrary and lookalike hosts`() { + assertEquals("Unsupported proxy host", validateProxyUrl("https://example.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://evilgooglevideo.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://googlevideo.com.example.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://example.googleusercontent.com/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://other.akamaized.net/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://127.0.0.1/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://[::1]/content")) + assertEquals("Unsupported proxy host", validateProxyUrl("https://i.ytimg.com.evil.example/content")) + assertEquals("URL credentials are not allowed", validateProxyUrl("https://evil.example@i.ytimg.com/content")) + } + + @Test + fun `rejects non-public ipv4 addresses`() { + val blocked = listOf( + "0.0.0.1", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.168.1.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + ) + blocked.forEach { assertFalse(isPublicProxyAddress(InetAddress.getByName(it)), it) } + assertTrue(isPublicProxyAddress(InetAddress.getByName("1.1.1.1"))) + } + + @Test + fun `rejects non-public ipv6 addresses`() { + val blocked = listOf( + "::", + "::1", + "fc00::1", + "fe80::1", + "2001:10::1", + "2001:20::1", + "2001:db8::1", + "2002:7f00:1::", + "ff02::1", + ) + blocked.forEach { assertFalse(isPublicProxyAddress(InetAddress.getByName(it)), it) } + assertTrue(isPublicProxyAddress(InetAddress.getByName("2606:4700:4700::1111"))) } } From e75f1819559530b6025e685f49f22ef5c4855088 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 00:31:29 +0200 Subject: [PATCH 05/22] test: cover proxy redirect and dns attacks --- .../server/OkHttpProxyServiceSecurityTest.kt | 66 +++++++ .../typetype/server/ProxyHttpExecutorTest.kt | 167 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt diff --git a/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt b/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt new file mode 100644 index 00000000..3c1a162e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/OkHttpProxyServiceSecurityTest.kt @@ -0,0 +1,66 @@ +package dev.typetype.server + +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.services.OkHttpProxyService +import kotlinx.coroutines.test.runTest +import okhttp3.Dns +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress + +class OkHttpProxyServiceSecurityTest { + @Test + fun `arbitrary destinations are rejected without a network call`() = runTest { + var calls = 0 + val client = OkHttpClient.Builder() + .addInterceptor { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("private".toResponseBody()) + .build() + } + .build() + + val result = OkHttpProxyService(client).pipe( + url = "https://example.com/collect", + rangeHeader = null, + domandBid = null, + ) + + assertEquals(ExtractionResult.BadRequest("Unsupported proxy host"), result) + assertEquals(0, calls) + } + + @Test + fun `supported media hosts remain available`() = runTest { + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("image".toResponseBody()) + .build() + } + .build() + + val result = OkHttpProxyService(client).pipe( + url = "https://i.ytimg.com/vi/id/hqdefault.jpg", + rangeHeader = null, + domandBid = null, + ) + + assertTrue(result is ExtractionResult.Success) + } +} diff --git a/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt b/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt new file mode 100644 index 00000000..9a46d449 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/ProxyHttpExecutorTest.kt @@ -0,0 +1,167 @@ +package dev.typetype.server + +import dev.typetype.server.services.ProxyHttpExecutor +import dev.typetype.server.services.ValidatingProxyDns +import okhttp3.Dns +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.net.InetAddress +import java.net.UnknownHostException + +class ProxyHttpExecutorTest { + @Test + fun `blocks private resolutions before sending a request`() { + var calls = 0 + val client = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("127.0.0.1")) }) + .addInterceptor { chain -> + calls += 1 + ok(chain) + } + .build() + + assertThrows(UnknownHostException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals(0, calls) + } + + @Test + fun `blocks a redirect outside the original provider`() { + var calls = 0 + val client = testClient { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "https://example.com/collect") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(IllegalArgumentException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Unsupported proxy host", error.message) + assertEquals(1, calls) + } + + @Test + fun `follows bounded redirects inside one provider`() { + val requestedHosts = mutableListOf() + val client = testClient { chain -> + requestedHosts += chain.request().url.host + if (requestedHosts.size == 1) { + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "https://yt3.ggpht.com/avatar") + .body("".toResponseBody()) + .build() + } else { + ok(chain) + } + } + + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")).use { response -> + assertTrue(response.isSuccessful) + } + assertEquals(listOf("i.ytimg.com", "yt3.ggpht.com"), requestedHosts) + } + + @Test + fun `rejects mixed public and private dns answers`() { + val dns = ValidatingProxyDns( + Dns { + listOf( + InetAddress.getByName("1.1.1.1"), + InetAddress.getByName("10.0.0.1"), + ) + }, + ) + + assertThrows(UnknownHostException::class.java) { dns.lookup("i.ytimg.com") } + } + + @Test + fun `rejects a later private dns rebind answer`() { + var lookups = 0 + val dns = ValidatingProxyDns( + Dns { + lookups += 1 + listOf(InetAddress.getByName(if (lookups == 1) "1.1.1.1" else "10.0.0.1")) + }, + ) + + assertEquals(listOf(InetAddress.getByName("1.1.1.1")), dns.lookup("i.ytimg.com")) + assertThrows(UnknownHostException::class.java) { dns.lookup("i.ytimg.com") } + } + + @Test + fun `rejects an https downgrade redirect`() { + val client = testClient { chain -> + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "http://i.ytimg.com/image.jpg") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(IllegalArgumentException::class.java) { + ProxyHttpExecutor(client).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Unsupported URL scheme: http", error.message) + } + + @Test + fun `stops a redirect loop at the configured bound`() { + var calls = 0 + val client = testClient { chain -> + calls += 1 + Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", "/next") + .body("".toResponseBody()) + .build() + } + + val error = assertThrows(java.io.IOException::class.java) { + ProxyHttpExecutor(client, maxRedirects = 2).execute(request("https://i.ytimg.com/image.jpg")) + } + assertEquals("Too many proxy redirects", error.message) + assertEquals(3, calls) + } + + private fun testClient(interceptor: Interceptor): OkHttpClient = OkHttpClient.Builder() + .dns(Dns { listOf(InetAddress.getByName("1.1.1.1")) }) + .addInterceptor(interceptor) + .build() + + private fun request(url: String): Request = Request.Builder().url(url).build() + + private fun ok(chain: Interceptor.Chain): Response = Response.Builder() + .request(chain.request()) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body("ok".toResponseBody()) + .build() +} From c646e1f5d2766be9a3862c918901bb1297311a08 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 12:14:13 +0200 Subject: [PATCH 06/22] fix: update vulnerable jsoup dependency --- build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index d4655639..3d141621 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -27,6 +27,11 @@ repositories { dependencies { implementation(platform("com.fasterxml.jackson:jackson-bom:2.22.1")) implementation(platform("io.netty:netty-bom:4.2.16.Final")) + constraints { + implementation("org.jsoup:jsoup:1.23.1") { + because("CVE-2026-71497 affects PipePipeExtractor's transitive jsoup version") + } + } implementation("io.ktor:ktor-server-core-jvm") implementation("io.ktor:ktor-server-netty-jvm") implementation("io.ktor:ktor-server-content-negotiation-jvm") From 25eb032f2e3893b7527ac21a09de9dbbfc28d49f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sat, 8 Aug 2026 12:50:04 +0200 Subject: [PATCH 07/22] chore: update server dependencies --- build.gradle.kts | 8 ++++---- gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 3d141621..406be47e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,7 +3,7 @@ import java.time.Instant plugins { kotlin("jvm") version "2.4.10" kotlin("plugin.serialization") version "2.4.10" - id("io.ktor.plugin") version "3.5.1" + id("io.ktor.plugin") version "3.5.2" id("jacoco") } @@ -48,14 +48,14 @@ dependencies { implementation("org.json:json:20260719") implementation("com.squareup.okhttp3:okhttp:5.4.0") implementation("io.lettuce:lettuce-core:7.6.0.RELEASE") - implementation("org.jetbrains.exposed:exposed-core:1.3.1") - implementation("org.jetbrains.exposed:exposed-jdbc:1.3.1") + implementation("org.jetbrains.exposed:exposed-core:1.4.0") + implementation("org.jetbrains.exposed:exposed-jdbc:1.4.0") implementation("com.zaxxer:HikariCP:7.1.0") implementation("org.postgresql:postgresql:42.7.13") implementation("org.xerial:sqlite-jdbc:3.53.2.1") implementation("com.password4j:password4j:1.8.4") implementation("com.auth0:java-jwt:4.6.0") - testImplementation("org.junit.jupiter:junit-jupiter:6.1.2") + testImplementation("org.junit.jupiter:junit-jupiter:6.1.3") testRuntimeOnly("org.junit.platform:junit-platform-launcher") testImplementation("io.mockk:mockk:1.14.11") testImplementation("io.ktor:ktor-server-test-host-jvm") diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a9db1155..69dd0d04 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.0-bin.zip networkTimeout=10000 retries=0 retryBackOffMs=500 From c32ad4162b4bab1a872f0b78edc1df53e08eef9e Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:20:24 +0200 Subject: [PATCH 08/22] fix: require explicit local extractor path --- CONTRIBUTING.md | 6 ++++++ settings.gradle.kts | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96bcc2bb..b5417bc3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,6 +44,12 @@ PipePipe Client and PipePipeExtractor are the behavioral references for extracti When a defect is general to PipePipeExtractor, prefer contributing the correction upstream. Keep TypeType-specific behavior in this repository only when it belongs to the TypeType API or when the upstream API cannot express the required backend behavior cleanly. +Builds use the PipePipeExtractor revision pinned in `build.gradle.kts`. To deliberately test a local checkout instead, pass its path explicitly: + +```sh +./gradlew -PpipePipeExtractorPath=../PipePipeExtractor test +``` + ## Programming preferences - Prefer clear names and structure over explanatory comments, but comments are welcome whenever a contributor finds them useful. diff --git a/settings.gradle.kts b/settings.gradle.kts index 459e1da9..0a997ae6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,7 +1,13 @@ rootProject.name = "typetype-server" -val localPipePipeExtractor = file("../PipePipeExtractor") -if (localPipePipeExtractor.isDirectory) { +val localPipePipeExtractor = providers.gradleProperty("pipePipeExtractorPath") + .orNull + ?.let { file(it) } + +if (localPipePipeExtractor != null) { + require(localPipePipeExtractor.isDirectory) { + "pipePipeExtractorPath must point to a PipePipeExtractor checkout" + } includeBuild(localPipePipeExtractor) { dependencySubstitution { substitute(module("com.github.InfinityLoop1308.PipePipeExtractor:extractor")) From 951bca9880e49c24bcd03f7e54f7152cec8e0e5d Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:26 +0200 Subject: [PATCH 09/22] fix: normalize YouTube channel tab URLs --- .../server/services/AllowedChannelsService.kt | 21 +++++++++++++++++++ .../services/BlockedContentProfileTest.kt | 1 + 2 files changed, 22 insertions(+) diff --git a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt index fba4aea1..222bb64a 100644 --- a/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AllowedChannelsService.kt @@ -11,6 +11,7 @@ import org.jetbrains.exposed.v1.core.or import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll +import java.net.URI class AllowedChannelsService { suspend fun getChannels(userId: String): List = DatabaseFactory.query { @@ -90,3 +91,23 @@ internal fun normalizeChannelKey(value: String): String = value.trim() Regex("^https://(?:www\\.|m\\.|music\\.)youtube\\.com", RegexOption.IGNORE_CASE), "https://youtube.com", ) + .withoutYoutubeTab() + +private fun String.withoutYoutubeTab(): String { + val uri = runCatching { URI(this) }.getOrNull() ?: return this + if (!uri.host.equals("youtube.com", ignoreCase = true)) return this + val segments = uri.path.split('/').filter(String::isNotBlank) + if (segments.size < 2 || segments.last().lowercase() !in YOUTUBE_CHANNEL_TABS) return this + val path = "/${segments.dropLast(1).joinToString("/")}" + return URI(uri.scheme, uri.userInfo, uri.host, uri.port, path, null, null).toString() +} + +private val YOUTUBE_CHANNEL_TABS = setOf( + "featured", + "videos", + "shorts", + "streams", + "playlists", + "community", + "about", +) diff --git a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt index 198aa1d3..29869968 100644 --- a/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt +++ b/src/test/kotlin/dev/typetype/server/services/BlockedContentProfileTest.kt @@ -32,6 +32,7 @@ class BlockedContentProfileTest { ) assertTrue(profile.blocksChannel("https://m.youtube.com/@Example/", "Other")) + assertTrue(profile.blocksChannel("https://youtube.com/@Example/streams", "Other")) assertTrue(profile.blocksChannel("", "test channel")) } From 83f53a16198131568e152969db408be3e9d315f1 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:36 +0200 Subject: [PATCH 10/22] feat: add RSS feed data model --- .../dev/typetype/server/db/DatabaseFactory.kt | 14 +++++ .../server/db/tables/RssFeedChannelsTable.kt | 9 +++ .../server/db/tables/RssFeedServicesTable.kt | 9 +++ .../server/db/tables/RssFeedsTable.kt | 26 ++++++++ .../server/db/tables/RssUserPoliciesTable.kt | 10 ++++ .../typetype/server/models/RssFeedModels.kt | 59 +++++++++++++++++++ .../dev/typetype/server/TestDatabase.kt | 8 +++ 7 files changed, 135 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index e99acb31..f964f6db 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -23,6 +23,10 @@ import dev.typetype.server.db.tables.AdminSettingsTable import dev.typetype.server.db.tables.AllowedChannelsTable import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.NotificationStatesTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -73,6 +77,10 @@ object DatabaseFactory { YoutubeSessionPairingsTable, BugReportsTable, NotificationStatesTable, + RssFeedsTable, + RssFeedChannelsTable, + RssFeedServicesTable, + RssUserPoliciesTable, ) exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS name TEXT") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS thumbnail_url TEXT") @@ -95,6 +103,12 @@ object DatabaseFactory { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS caption_styles TEXT NOT NULL DEFAULT '{}'") exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS access_mode TEXT NOT NULL DEFAULT 'unrestricted'") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_enabled BOOLEAN NOT NULL DEFAULT FALSE") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_public_base_url TEXT") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_max_feeds_per_user INTEGER NOT NULL DEFAULT 10") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_max_items INTEGER NOT NULL DEFAULT 50") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_minimum_poll_minutes INTEGER NOT NULL DEFAULT 5") + exec("ALTER TABLE admin_settings ADD COLUMN IF NOT EXISTS rss_rate_limit_per_minute INTEGER NOT NULL DEFAULT 30") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") exec("ALTER TABLE blocked_channels ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'user'") exec("ALTER TABLE blocked_videos ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT ''") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt new file mode 100644 index 00000000..e7cb052c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedChannelsTable.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedChannelsTable : Table("rss_feed_channels") { + val feedId = text("feed_id") + val channelUrl = text("channel_url") + override val primaryKey = PrimaryKey(feedId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt new file mode 100644 index 00000000..b02f88b2 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedServicesTable.kt @@ -0,0 +1,9 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedServicesTable : Table("rss_feed_services") { + val feedId = text("feed_id") + val serviceId = integer("service_id") + override val primaryKey = PrimaryKey(feedId, serviceId) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt new file mode 100644 index 00000000..37c69f5c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssFeedsTable.kt @@ -0,0 +1,26 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssFeedsTable : Table("rss_feeds") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val tokenHash = text("token_hash") + val scope = text("scope") + val includeVideos = bool("include_videos") + val includeShorts = bool("include_shorts") + val includeLive = bool("include_live") + val includeUpcoming = bool("include_upcoming") + val enabled = bool("enabled").default(true) + val createdAt = long("created_at") + val updatedAt = long("updated_at") + val lastUsedAt = long("last_used_at").nullable() + + init { + index(false, userId) + index(false, createdAt) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt new file mode 100644 index 00000000..26994cca --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/RssUserPoliciesTable.kt @@ -0,0 +1,10 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object RssUserPoliciesTable : Table("rss_user_policies") { + val userId = text("user_id") + val enabled = bool("enabled").default(true) + val updatedAt = long("updated_at") + override val primaryKey = PrimaryKey(userId) +} diff --git a/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt b/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt new file mode 100644 index 00000000..e3ebbb5f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/RssFeedModels.kt @@ -0,0 +1,59 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class RssFeedRequest( + val name: String, + val scope: String = "all", + val channelUrls: List = emptyList(), + val serviceIds: List = listOf(0, 5, 6), + val includeVideos: Boolean = true, + val includeShorts: Boolean = true, + val includeLive: Boolean = true, + val includeUpcoming: Boolean = true, +) + +@Serializable +data class RssFeedItem( + val id: String, + val name: String, + val scope: String, + val channelUrls: List, + val serviceIds: List, + val includeVideos: Boolean, + val includeShorts: Boolean, + val includeLive: Boolean, + val includeUpcoming: Boolean, + val enabled: Boolean, + val createdAt: Long, + val updatedAt: Long, + val lastUsedAt: Long? = null, +) + +@Serializable +data class RssFeedSecretItem(val feed: RssFeedItem, val feedUrl: String) + +@Serializable +data class RssFeedEnabledRequest(val enabled: Boolean) + +@Serializable +data class RssUserPolicyRequest(val enabled: Boolean) + +@Serializable +data class AdminRssFeedItem( + val feed: RssFeedItem, + val userId: String, + val userName: String, + val userEmail: String, + val userRssEnabled: Boolean, + val userSuspended: Boolean, +) + +@Serializable +data class AdminRssFeedsPage( + val items: List, + val page: Int, + val limit: Int, + val total: Long, +) diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index f55367ae..1a22cc72 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -14,6 +14,10 @@ import dev.typetype.server.db.tables.PasswordResetTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.ProgressTable +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable import dev.typetype.server.db.tables.SavedPlaylistsTable import dev.typetype.server.db.tables.NotificationStatesTable import dev.typetype.server.db.tables.SearchHistoryTable @@ -88,6 +92,10 @@ object TestDatabase { value?.takeIf { it.isNotBlank() } ?: fallback fun truncateAll() = transaction { + RssFeedChannelsTable.deleteAll() + RssFeedServicesTable.deleteAll() + RssFeedsTable.deleteAll() + RssUserPoliciesTable.deleteAll() PlaylistVideosTable.deleteAll() PlaylistsTable.deleteAll() SavedPlaylistsTable.deleteAll() From a4fc41c76adcd1775319333004faf58ea5471cd8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:40 +0200 Subject: [PATCH 11/22] feat: add RSS feed persistence --- .../server/services/RssFeedRepository.kt | 178 ++++++++++++++++++ .../server/services/RssFeedRowMapper.kt | 31 +++ .../typetype/server/services/RssFeedSecret.kt | 23 +++ .../server/services/RssFeedSelections.kt | 23 +++ 4 files changed, 255 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt new file mode 100644 index 00000000..d8fab751 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedRepository.kt @@ -0,0 +1,178 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.RssFeedRequest +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.jetbrains.exposed.v1.jdbc.update + +internal data class StoredRssFeed(val item: RssFeedItem, val userId: String, val tokenHash: String) + +internal class RssFeedRepository { + suspend fun list(userId: String): List = DatabaseFactory.query { + val rows = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId } + .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) + .toList() + val selections = loadRssFeedSelections(rows.map { it[RssFeedsTable.id] }) + rows.map { it.toStoredFeed(selections).item } + } + + suspend fun find(feedId: String): StoredRssFeed? = DatabaseFactory.query { + RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.singleOrNull()?.toStoredFeed() + } + + suspend fun createWithinLimit( + userId: String, + id: String, + tokenHash: String, + request: RssFeedRequest, + limit: Int, + ): RssFeedItem? = + DatabaseFactory.query { + TransactionManager.current().exec("SELECT pg_advisory_xact_lock(${userId.hashCode().toLong()})") + val count = RssFeedsTable.selectAll().where { RssFeedsTable.userId eq userId }.count() + if (count >= limit) return@query null + val now = System.currentTimeMillis() + RssFeedsTable.insert { + it[RssFeedsTable.id] = id + it[RssFeedsTable.userId] = userId + it[name] = request.name + it[RssFeedsTable.tokenHash] = tokenHash + it[scope] = request.scope + it[includeVideos] = request.includeVideos + it[includeShorts] = request.includeShorts + it[includeLive] = request.includeLive + it[includeUpcoming] = request.includeUpcoming + it[enabled] = true + it[createdAt] = now + it[updatedAt] = now + } + replaceSelections(id, request) + RssFeedsTable.selectAll().where { RssFeedsTable.id eq id }.single().toStoredFeed().item + } + + suspend fun update(userId: String, feedId: String, request: RssFeedRequest): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[name] = request.name + it[scope] = request.scope + it[includeVideos] = request.includeVideos + it[includeShorts] = request.includeShorts + it[includeLive] = request.includeLive + it[includeUpcoming] = request.includeUpcoming + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) return@query null + replaceSelections(feedId, request) + RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun replaceToken(userId: String, feedId: String, tokenHash: String): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[RssFeedsTable.tokenHash] = tokenHash + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun setEnabled(userId: String, feedId: String, enabled: Boolean): RssFeedItem? = + DatabaseFactory.query { + val changed = RssFeedsTable.update({ + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }) { + it[RssFeedsTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun setEnabledByAdmin(feedId: String, enabled: Boolean): RssFeedItem? = DatabaseFactory.query { + val changed = RssFeedsTable.update({ RssFeedsTable.id eq feedId }) { + it[RssFeedsTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) null else RssFeedsTable.selectAll() + .where { RssFeedsTable.id eq feedId }.single().toStoredFeed().item + } + + suspend fun delete(userId: String, feedId: String): Boolean = DatabaseFactory.query { + val owned = RssFeedsTable.selectAll().where { + (RssFeedsTable.id eq feedId) and (RssFeedsTable.userId eq userId) + }.count() > 0 + if (!owned) return@query false + deleteFeed(feedId) + true + } + + suspend fun deleteByAdmin(feedId: String): Boolean = DatabaseFactory.query { + val exists = RssFeedsTable.selectAll().where { RssFeedsTable.id eq feedId }.count() > 0 + if (!exists) return@query false + deleteFeed(feedId) + true + } + + suspend fun touch(feedId: String, timestamp: Long) = DatabaseFactory.query { + RssFeedsTable.update({ RssFeedsTable.id eq feedId }) { it[lastUsedAt] = timestamp } + } + + suspend fun userEnabled(userId: String): Boolean = DatabaseFactory.query { + val active = UsersTable.selectAll().where { UsersTable.id eq userId } + .singleOrNull()?.get(UsersTable.suspended) == false + if (!active) return@query false + RssUserPoliciesTable.selectAll().where { RssUserPoliciesTable.userId eq userId } + .singleOrNull()?.get(RssUserPoliciesTable.enabled) ?: true + } + + suspend fun setUserEnabled(userId: String, enabled: Boolean) = DatabaseFactory.query { + val changed = RssUserPoliciesTable.update({ RssUserPoliciesTable.userId eq userId }) { + it[RssUserPoliciesTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + if (changed == 0) RssUserPoliciesTable.insert { + it[RssUserPoliciesTable.userId] = userId + it[RssUserPoliciesTable.enabled] = enabled + it[updatedAt] = System.currentTimeMillis() + } + } + + private fun replaceSelections(feedId: String, request: RssFeedRequest) { + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + request.channelUrls.forEach { url -> + RssFeedChannelsTable.insert { + it[RssFeedChannelsTable.feedId] = feedId + it[channelUrl] = url + } + } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + request.serviceIds.forEach { service -> + RssFeedServicesTable.insert { + it[RssFeedServicesTable.feedId] = feedId + it[serviceId] = service + } + } + } + + private fun deleteFeed(feedId: String) { + RssFeedChannelsTable.deleteWhere { RssFeedChannelsTable.feedId eq feedId } + RssFeedServicesTable.deleteWhere { RssFeedServicesTable.feedId eq feedId } + RssFeedsTable.deleteWhere { RssFeedsTable.id eq feedId } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt new file mode 100644 index 00000000..55421556 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedRowMapper.kt @@ -0,0 +1,31 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.models.RssFeedItem +import org.jetbrains.exposed.v1.core.ResultRow + +internal fun ResultRow.toStoredFeed(): StoredRssFeed = + toStoredFeed(loadRssFeedSelections(listOf(this[RssFeedsTable.id]))) + +internal fun ResultRow.toStoredFeed(selections: RssFeedSelections): StoredRssFeed { + val id = this[RssFeedsTable.id] + return StoredRssFeed( + item = RssFeedItem( + id = id, + name = this[RssFeedsTable.name], + scope = this[RssFeedsTable.scope], + channelUrls = selections.channels[id].orEmpty(), + serviceIds = selections.services[id].orEmpty(), + includeVideos = this[RssFeedsTable.includeVideos], + includeShorts = this[RssFeedsTable.includeShorts], + includeLive = this[RssFeedsTable.includeLive], + includeUpcoming = this[RssFeedsTable.includeUpcoming], + enabled = this[RssFeedsTable.enabled], + createdAt = this[RssFeedsTable.createdAt], + updatedAt = this[RssFeedsTable.updatedAt], + lastUsedAt = this[RssFeedsTable.lastUsedAt], + ), + userId = this[RssFeedsTable.userId], + tokenHash = this[RssFeedsTable.tokenHash], + ) +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt new file mode 100644 index 00000000..563dda78 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedSecret.kt @@ -0,0 +1,23 @@ +package dev.typetype.server.services + +import java.security.MessageDigest +import java.security.SecureRandom +import java.util.Base64 + +internal class RssFeedSecret(private val random: SecureRandom = SecureRandom()) { + fun create(): String { + val bytes = ByteArray(32) + random.nextBytes(bytes) + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) + } + + fun hash(secret: String): String = Base64.getUrlEncoder().withoutPadding().encodeToString( + MessageDigest.getInstance("SHA-256").digest(secret.toByteArray(Charsets.UTF_8)), + ) + + fun matches(secret: String, expectedHash: String): Boolean { + val actual = runCatching { Base64.getUrlDecoder().decode(hash(secret)) }.getOrNull() ?: return false + val expected = runCatching { Base64.getUrlDecoder().decode(expectedHash) }.getOrNull() ?: return false + return MessageDigest.isEqual(actual, expected) + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt new file mode 100644 index 00000000..17343952 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedSelections.kt @@ -0,0 +1,23 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.RssFeedChannelsTable +import dev.typetype.server.db.tables.RssFeedServicesTable +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal data class RssFeedSelections( + val channels: Map>, + val services: Map>, +) + +internal fun loadRssFeedSelections(feedIds: List): RssFeedSelections { + if (feedIds.isEmpty()) return RssFeedSelections(emptyMap(), emptyMap()) + val channels = RssFeedChannelsTable.selectAll() + .where { RssFeedChannelsTable.feedId inList feedIds } + .groupBy({ it[RssFeedChannelsTable.feedId] }, { it[RssFeedChannelsTable.channelUrl] }) + val services = RssFeedServicesTable.selectAll() + .where { RssFeedServicesTable.feedId inList feedIds } + .groupBy({ it[RssFeedServicesTable.feedId] }, { it[RssFeedServicesTable.serviceId] }) + .mapValues { (_, values) -> values.sorted() } + return RssFeedSelections(channels, services) +} From bae862e255b824f4aaf96d286d3d6ad16542f4a0 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:46 +0200 Subject: [PATCH 12/22] feat: configure RSS instance policy --- .../server/db/tables/AdminSettingsTable.kt | 6 ++ .../server/models/AdminSettingsItem.kt | 6 ++ .../server/models/InstanceResponse.kt | 10 +++ .../server/services/AdminSettingsService.kt | 62 ++++++++++++++++--- .../server/services/InstanceService.kt | 8 +++ .../server/AdminSettingsDefaultsTest.kt | 23 +++++++ .../dev/typetype/server/InstanceRoutesTest.kt | 15 +++++ 7 files changed, 122 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt index 6590b798..e0703618 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/AdminSettingsTable.kt @@ -18,5 +18,11 @@ object AdminSettingsTable : Table("admin_settings") { val oidcAutoRedirect = bool("oidc_auto_redirect").default(false) val youtubeRemoteLoginEnabled = bool("youtube_remote_login_enabled").default(false) val accessMode = text("access_mode").default("unrestricted") + val rssEnabled = bool("rss_enabled").default(false) + val rssPublicBaseUrl = text("rss_public_base_url").nullable() + val rssMaxFeedsPerUser = integer("rss_max_feeds_per_user").default(10) + val rssMaxItems = integer("rss_max_items").default(50) + val rssMinimumPollMinutes = integer("rss_minimum_poll_minutes").default(5) + val rssRateLimitPerMinute = integer("rss_rate_limit_per_minute").default(30) override val primaryKey = PrimaryKey(id) } diff --git a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt index b58d80c1..08b60e63 100644 --- a/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/AdminSettingsItem.kt @@ -18,4 +18,10 @@ data class AdminSettingsItem( val oidcAutoRedirect: Boolean = false, val youtubeRemoteLoginEnabled: Boolean = false, val accessMode: String = "unrestricted", + val rssEnabled: Boolean = false, + val rssPublicBaseUrl: String? = null, + val rssMaxFeedsPerUser: Int = 10, + val rssMaxItems: Int = 50, + val rssMinimumPollMinutes: Int = 5, + val rssRateLimitPerMinute: Int = 30, ) diff --git a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt index 2ec0c25a..55d8916d 100644 --- a/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt +++ b/src/main/kotlin/dev/typetype/server/models/InstanceResponse.kt @@ -24,4 +24,14 @@ data class InstanceResponse( val youtubeRemoteLoginEnabled: Boolean = false, val youtubeRemoteLoginReady: Boolean = false, val youtubeRemoteLoginUnavailableReason: String? = null, + val rss: RssInstanceCapability = RssInstanceCapability(), +) + +@Serializable +data class RssInstanceCapability( + val enabled: Boolean = false, + val maxFeedsPerUser: Int = 0, + val maxItems: Int = 0, + val minimumPollMinutes: Int = 0, + val rateLimitPerMinute: Int = 0, ) diff --git a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt b/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt index 16495092..7a761a05 100644 --- a/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/AdminSettingsService.kt @@ -8,6 +8,7 @@ import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.update +import java.net.URI private const val SETTINGS_ROW_ID = 1 @@ -33,6 +34,12 @@ class AdminSettingsService( oidcAutoRedirect = it[AdminSettingsTable.oidcAutoRedirect], youtubeRemoteLoginEnabled = it[AdminSettingsTable.youtubeRemoteLoginEnabled], accessMode = it[AdminSettingsTable.accessMode].toAccessMode(), + rssEnabled = it[AdminSettingsTable.rssEnabled], + rssPublicBaseUrl = it[AdminSettingsTable.rssPublicBaseUrl], + rssMaxFeedsPerUser = it[AdminSettingsTable.rssMaxFeedsPerUser], + rssMaxItems = it[AdminSettingsTable.rssMaxItems], + rssMinimumPollMinutes = it[AdminSettingsTable.rssMinimumPollMinutes], + rssRateLimitPerMinute = it[AdminSettingsTable.rssRateLimitPerMinute], ).normalized() } ?: defaultSettings().normalized() } @@ -59,6 +66,12 @@ class AdminSettingsService( it[oidcAutoRedirect] = settings.oidcAutoRedirect it[youtubeRemoteLoginEnabled] = settings.youtubeRemoteLoginEnabled it[accessMode] = settings.accessMode.toAccessMode() + it[rssEnabled] = settings.rssEnabled + it[rssPublicBaseUrl] = settings.rssPublicBaseUrl + it[rssMaxFeedsPerUser] = settings.rssMaxFeedsPerUser + it[rssMaxItems] = settings.rssMaxItems + it[rssMinimumPollMinutes] = settings.rssMinimumPollMinutes + it[rssRateLimitPerMinute] = settings.rssRateLimitPerMinute } } else { AdminSettingsTable.insert { @@ -76,6 +89,12 @@ class AdminSettingsService( it[oidcAutoRedirect] = settings.oidcAutoRedirect it[youtubeRemoteLoginEnabled] = settings.youtubeRemoteLoginEnabled it[accessMode] = settings.accessMode.toAccessMode() + it[rssEnabled] = settings.rssEnabled + it[rssPublicBaseUrl] = settings.rssPublicBaseUrl + it[rssMaxFeedsPerUser] = settings.rssMaxFeedsPerUser + it[rssMaxItems] = settings.rssMaxItems + it[rssMinimumPollMinutes] = settings.rssMinimumPollMinutes + it[rssRateLimitPerMinute] = settings.rssRateLimitPerMinute } } } @@ -83,17 +102,44 @@ class AdminSettingsService( return settings } - private fun AdminSettingsItem.normalized(): AdminSettingsItem = copy( - name = name.trim().takeIf { it.isNotEmpty() } ?: DEFAULT_INSTANCE_NAME, - tagline = tagline.normalizeOptionalText(), - logoUrl = logoUrl.normalizeOptionalText(), - bannerUrl = bannerUrl.normalizeOptionalText(), - minAndroidClientVersion = minAndroidClientVersion.normalizeOptionalText(), - accessMode = accessMode.toAccessMode(), - ) + private fun AdminSettingsItem.normalized(): AdminSettingsItem { + val publicBaseUrl = rssPublicBaseUrl.normalizePublicBaseUrl() + require(!rssEnabled || publicBaseUrl != null) { + "RSS public base URL is required when RSS is enabled" + } + return copy( + name = name.trim().takeIf { it.isNotEmpty() } ?: DEFAULT_INSTANCE_NAME, + tagline = tagline.normalizeOptionalText(), + logoUrl = logoUrl.normalizeOptionalText(), + bannerUrl = bannerUrl.normalizeOptionalText(), + minAndroidClientVersion = minAndroidClientVersion.normalizeOptionalText(), + accessMode = accessMode.toAccessMode(), + rssPublicBaseUrl = publicBaseUrl, + rssMaxFeedsPerUser = rssMaxFeedsPerUser.coerceIn(1, 100), + rssMaxItems = rssMaxItems.coerceIn(1, 200), + rssMinimumPollMinutes = rssMinimumPollMinutes.coerceIn(1, 1_440), + rssRateLimitPerMinute = rssRateLimitPerMinute.coerceIn(1, 600), + ) + } private fun String?.normalizeOptionalText(): String? = this?.trim()?.takeIf { it.isNotEmpty() } + private fun String?.normalizePublicBaseUrl(): String? { + val value = normalizeOptionalText() ?: return null + val uri = runCatching { URI(value) }.getOrNull() + ?: throw IllegalArgumentException("RSS public base URL must be an absolute HTTP or HTTPS URL") + require(uri.scheme in setOf("http", "https") && !uri.host.isNullOrBlank()) { + "RSS public base URL must be an absolute HTTP or HTTPS URL" + } + require(uri.rawQuery == null && uri.rawFragment == null) { + "RSS public base URL cannot contain a query or fragment" + } + require(uri.rawUserInfo == null) { + "RSS public base URL cannot contain credentials" + } + return value.trimEnd('/') + } + companion object { @Volatile private var cachedSettings: AdminSettingsItem? = null diff --git a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt index 07e6bfa7..3225e2d6 100644 --- a/src/main/kotlin/dev/typetype/server/services/InstanceService.kt +++ b/src/main/kotlin/dev/typetype/server/services/InstanceService.kt @@ -7,6 +7,7 @@ import dev.typetype.server.models.InstanceMinClientVersion import dev.typetype.server.models.InstanceResponse import dev.typetype.server.models.OidcPublicConfig import dev.typetype.server.models.YoutubeRemoteLoginStatus +import dev.typetype.server.models.RssInstanceCapability class InstanceService( private val authService: AuthService, @@ -47,6 +48,13 @@ class InstanceService( youtubeRemoteLoginEnabled = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginReady = youtubeRemoteLoginStatus.ready, youtubeRemoteLoginUnavailableReason = youtubeRemoteLoginStatus.unavailableReason, + rss = RssInstanceCapability( + enabled = settings.rssEnabled && settings.rssPublicBaseUrl != null, + maxFeedsPerUser = settings.rssMaxFeedsPerUser, + maxItems = settings.rssMaxItems, + minimumPollMinutes = settings.rssMinimumPollMinutes, + rateLimitPerMinute = settings.rssRateLimitPerMinute, + ), ) } diff --git a/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt b/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt index bc0d8def..fc35b7fa 100644 --- a/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt +++ b/src/test/kotlin/dev/typetype/server/AdminSettingsDefaultsTest.kt @@ -4,6 +4,7 @@ import dev.typetype.server.models.AdminSettingsItem import dev.typetype.server.services.AdminSettingsService import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -27,4 +28,26 @@ class AdminSettingsDefaultsTest { assertEquals(false, settings.get().youtubeRemoteLoginEnabled) } + + @Test + fun `RSS public URL rejects embedded credentials`() = runTest { + val error = runCatching { + AdminSettingsService().upsert( + AdminSettingsItem(rssPublicBaseUrl = "https://user:password@video.example"), + ) + }.exceptionOrNull() + + assertTrue(error is IllegalArgumentException) + assertEquals("RSS public base URL cannot contain credentials", error?.message) + } + + @Test + fun `RSS cannot be enabled without a public URL`() = runTest { + val error = runCatching { + AdminSettingsService().upsert(AdminSettingsItem(rssEnabled = true)) + }.exceptionOrNull() + + assertTrue(error is IllegalArgumentException) + assertEquals("RSS public base URL is required when RSS is enabled", error?.message) + } } diff --git a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt index 3881e19a..7d2eeb22 100644 --- a/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/InstanceRoutesTest.kt @@ -73,6 +73,9 @@ class InstanceRoutesTest { assertEquals(false, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(false, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals("disabled", root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) + val rss = root["rss"]?.jsonObject + assertEquals(false, rss?.get("enabled")?.jsonPrimitive?.boolean) + assertEquals(10, rss?.get("maxFeedsPerUser")?.jsonPrimitive?.int) assertEquals(listOf(0, 3, 4, 5, 6), root["supportedServices"]?.jsonArray?.map { it.jsonPrimitive.int }) assertEquals(null, root["androidPlayback"]) } @@ -90,6 +93,12 @@ class InstanceRoutesTest { localLoginEnabled = false, oidcAutoRedirect = true, youtubeRemoteLoginEnabled = true, + rssEnabled = true, + rssPublicBaseUrl = "https://video.example/", + rssMaxFeedsPerUser = 4, + rssMaxItems = 80, + rssMinimumPollMinutes = 15, + rssRateLimitPerMinute = 12, ) ) val auth = AuthService.fixed(TEST_USER_ID, hasUsers = true) @@ -115,6 +124,12 @@ class InstanceRoutesTest { assertEquals(true, root["youtubeRemoteLoginEnabled"]?.jsonPrimitive?.boolean) assertEquals(true, root["youtubeRemoteLoginReady"]?.jsonPrimitive?.boolean) assertEquals(null, root["youtubeRemoteLoginUnavailableReason"]?.jsonPrimitive?.contentOrNull) + val rss = root["rss"]!!.jsonObject + assertEquals(true, rss["enabled"]?.jsonPrimitive?.boolean) + assertEquals(4, rss["maxFeedsPerUser"]?.jsonPrimitive?.int) + assertEquals(80, rss["maxItems"]?.jsonPrimitive?.int) + assertEquals(15, rss["minimumPollMinutes"]?.jsonPrimitive?.int) + assertEquals(12, rss["rateLimitPerMinute"]?.jsonPrimitive?.int) val register = client.post("/auth/register") { contentType(ContentType.Application.Json) setBody("""{"email":"new@test.local","password":"secret","name":"New"}""") From a5d277cc2d8b1f239cab27a5a9c8632dd3faa5a8 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:49 +0200 Subject: [PATCH 13/22] feat: manage private RSS feeds --- .../server/services/RssFeedAdminRepository.kt | 46 ++++++ .../services/RssFeedManagementService.kt | 135 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt new file mode 100644 index 00000000..76816a96 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedAdminRepository.kt @@ -0,0 +1,46 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.RssFeedsTable +import dev.typetype.server.db.tables.RssUserPoliciesTable +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminRssFeedItem +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll + +internal class RssFeedAdminRepository { + suspend fun list(page: Int, limit: Int): Pair, Long> = DatabaseFactory.query { + val total = RssFeedsTable.selectAll().count() + val rows = RssFeedsTable.selectAll() + .orderBy(RssFeedsTable.createdAt to SortOrder.DESC) + .limit(limit) + .offset((page - 1L) * limit) + .toList() + val selections = loadRssFeedSelections(rows.map { it[RssFeedsTable.id] }) + val feeds = rows.map { it.toStoredFeed(selections) } + val userIds = feeds.map { it.userId }.distinct() + val users = if (userIds.isEmpty()) emptyMap() else UsersTable.selectAll() + .where { UsersTable.id inList userIds } + .associateBy { it[UsersTable.id] } + val policies = if (userIds.isEmpty()) emptyMap() else RssUserPoliciesTable.selectAll() + .where { RssUserPoliciesTable.userId inList userIds } + .associate { it[RssUserPoliciesTable.userId] to it[RssUserPoliciesTable.enabled] } + feeds.mapNotNull { stored -> + val user = users[stored.userId] ?: return@mapNotNull null + AdminRssFeedItem( + feed = stored.item, + userId = stored.userId, + userName = user[UsersTable.name], + userEmail = user[UsersTable.email], + userRssEnabled = policies[stored.userId] ?: true, + userSuspended = user[UsersTable.suspended], + ) + } to total + } + + suspend fun userExists(userId: String): Boolean = DatabaseFactory.query { + UsersTable.selectAll().where { UsersTable.id eq userId }.any() + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt new file mode 100644 index 00000000..868f6a30 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedManagementService.kt @@ -0,0 +1,135 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.AdminRssFeedsPage +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.models.RssFeedSecretItem +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.util.UUID + +class RssFeedManagementService internal constructor( + private val settings: AdminSettingsService, + private val subscriptions: SubscriptionsService, + private val repository: RssFeedRepository = RssFeedRepository(), + private val adminRepository: RssFeedAdminRepository = RssFeedAdminRepository(), + private val secrets: RssFeedSecret = RssFeedSecret(), +) { + suspend fun list(userId: String): List { + requireAvailable(userId) + return repository.list(userId) + } + + suspend fun create(userId: String, request: RssFeedRequest): RssFeedSecretItem { + val config = requireAvailable(userId) + val normalized = normalize(userId, request) + val secret = secrets.create() + val feed = repository.createWithinLimit( + userId, + UUID.randomUUID().toString(), + secrets.hash(secret), + normalized, + config.rssMaxFeedsPerUser, + ) ?: throw RssFeedException("RSS feed limit reached", "rss_feed_limit_reached") + return RssFeedSecretItem(feed, feedUrl(config.rssPublicBaseUrl!!, feed.id, secret)) + } + + suspend fun update(userId: String, feedId: String, request: RssFeedRequest): RssFeedItem { + requireAvailable(userId) + val normalized = normalize(userId, request) + return repository.update(userId, feedId, normalized) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun setEnabled(userId: String, feedId: String, enabled: Boolean): RssFeedItem { + requireAvailable(userId) + return repository.setEnabled(userId, feedId, enabled) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun regenerate(userId: String, feedId: String): RssFeedSecretItem { + val config = requireAvailable(userId) + val secret = secrets.create() + val feed = repository.replaceToken(userId, feedId, secrets.hash(secret)) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + return RssFeedSecretItem(feed, feedUrl(config.rssPublicBaseUrl!!, feed.id, secret)) + } + + suspend fun delete(userId: String, feedId: String) { + requireAvailable(userId) + if (!repository.delete(userId, feedId)) throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun adminList(page: Int, limit: Int): AdminRssFeedsPage { + val (items, total) = adminRepository.list(page, limit) + return AdminRssFeedsPage(items, page, limit, total) + } + + suspend fun adminSetEnabled(feedId: String, enabled: Boolean): RssFeedItem = + repository.setEnabledByAdmin(feedId, enabled) + ?: throw RssFeedException("RSS feed not found", "rss_feed_not_found") + + suspend fun adminDelete(feedId: String) { + if (!repository.deleteByAdmin(feedId)) throw RssFeedException("RSS feed not found", "rss_feed_not_found") + } + + suspend fun adminSetUserEnabled(userId: String, enabled: Boolean) { + if (!adminRepository.userExists(userId)) { + throw RssFeedException("User not found", "rss_user_not_found") + } + repository.setUserEnabled(userId, enabled) + } + + private suspend fun requireAvailable(userId: String) = settings.get().also { config -> + if (!config.rssEnabled || config.rssPublicBaseUrl == null) { + throw RssFeedException("RSS feeds are disabled", "rss_disabled") + } + if (!repository.userEnabled(userId)) throw RssFeedException("RSS feeds are disabled for this account", "rss_user_disabled") + } + + private suspend fun normalize(userId: String, request: RssFeedRequest): RssFeedRequest { + val name = request.name.trim() + if (name.length !in 1..100) throw RssFeedException("Name must contain 1 to 100 characters", "rss_invalid_name") + if (request.scope !in SCOPES) throw RssFeedException("Invalid RSS scope", "rss_invalid_scope") + val services = request.serviceIds.distinct().sorted() + if (services.isEmpty() || services.any { it !in SERVICES }) { + throw RssFeedException("Select at least one supported service", "rss_invalid_services") + } + if (!request.hasSelectedType()) throw RssFeedException("Select at least one content type", "rss_invalid_types") + val channels = when (request.scope) { + "all" -> emptyList() + "channels" -> validateChannels(userId, request.channelUrls) + else -> throw RssFeedException("Invalid RSS scope", "rss_invalid_scope") + } + return request.copy( + name = name, + channelUrls = channels, + serviceIds = services, + ) + } + + private suspend fun validateChannels(userId: String, rawChannels: List): List { + val channels = rawChannels.map(ChannelUrlCanonicalizer::canonicalize).filter(String::isNotBlank).distinct() + if (channels.isEmpty() || channels.size > 100) { + throw RssFeedException("Select between 1 and 100 subscribed channels", "rss_invalid_channels") + } + val subscribed = subscriptions.getAll(userId).map { it.channelUrl }.toSet() + if (channels.any { it !in subscribed }) { + throw RssFeedException("RSS channels must belong to your subscriptions", "rss_channel_not_subscribed") + } + return channels.sorted() + } + + private fun feedUrl(baseUrl: String, feedId: String, secret: String): String = + "$baseUrl/api/rss/feeds/$feedId.xml?token=${URLEncoder.encode(secret, StandardCharsets.UTF_8)}" + + private fun RssFeedRequest.hasSelectedType(): Boolean = + includeVideos || includeShorts || includeLive || includeUpcoming + + companion object { + private val SCOPES = setOf("all", "channels") + private val SERVICES = setOf(0, 5, 6) + } +} + +class RssFeedException(message: String, val code: String) : IllegalArgumentException(message) From c2d06fc6b6f89c64ac5819c79bc45b422f41fdbe Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:53 +0200 Subject: [PATCH 14/22] feat: render private RSS feeds --- .../server/services/RssDocumentRenderer.kt | 67 +++++++++++++++++++ .../server/services/RssFeedReaderService.kt | 60 +++++++++++++++++ .../server/services/RssFeedThrottle.kt | 29 ++++++++ .../server/services/RssVideoMetadata.kt | 20 ++++++ .../server/services/RssVideoTypeFilter.kt | 16 +++++ .../services/SubscriptionFeedService.kt | 3 + 6 files changed, 195 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt diff --git a/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt new file mode 100644 index 00000000..c84e9665 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssDocumentRenderer.kt @@ -0,0 +1,67 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem +import java.io.ByteArrayOutputStream +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import javax.xml.stream.XMLOutputFactory +import javax.xml.stream.XMLStreamWriter + +internal object RssDocumentRenderer { + fun render( + feed: RssFeedItem, + videos: List, + publicBaseUrl: String, + lastModified: Long, + ): ByteArray { + val output = ByteArrayOutputStream() + val writer = XMLOutputFactory.newFactory().createXMLStreamWriter(output, StandardCharsets.UTF_8.name()) + writer.writeStartDocument(StandardCharsets.UTF_8.name(), "1.0") + writer.writeStartElement("rss") + writer.writeAttribute("version", "2.0") + writer.writeStartElement("channel") + writer.element("title", feed.name) + writer.element("link", publicBaseUrl) + writer.element("description", "TypeType subscription feed: ${feed.name}") + writer.element("generator", "TypeType") + writer.element("lastBuildDate", RFC_1123.format(Instant.ofEpochMilli(lastModified))) + videos.forEach { writer.item(it, publicBaseUrl) } + writer.writeEndElement() + writer.writeEndElement() + writer.writeEndDocument() + writer.close() + return output.toByteArray() + } + + fun lastModified(feed: RssFeedItem, videos: List, now: Long): Long = + maxOf(feed.updatedAt, videos.maxOfOrNull(RssVideoMetadata::publishedAtMillis) ?: feed.updatedAt) + .coerceAtMost(now) + + private fun XMLStreamWriter.item(video: VideoItem, publicBaseUrl: String) { + val watchUrl = "$publicBaseUrl/watch?v=${URLEncoder.encode(video.url, StandardCharsets.UTF_8)}" + writeStartElement("item") + element("title", video.title) + element("link", watchUrl) + writeStartElement("guid") + writeAttribute("isPermaLink", "false") + writeCharacters("${RssVideoMetadata.serviceId(video)}:${video.id}") + writeEndElement() + element("author", video.uploaderName) + video.shortDescription?.takeIf(String::isNotBlank)?.let { element("description", it) } + RssVideoMetadata.publishedAtMillis(video).takeIf { it > 0 } + ?.let { element("pubDate", RFC_1123.format(Instant.ofEpochMilli(it))) } + writeEndElement() + } + + private fun XMLStreamWriter.element(name: String, value: String) { + writeStartElement(name) + writeCharacters(value) + writeEndElement() + } + + private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt new file mode 100644 index 00000000..eae437df --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedReaderService.kt @@ -0,0 +1,60 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem +import java.security.MessageDigest +import java.util.HexFormat + +class RssFeedReaderService internal constructor( + private val settings: AdminSettingsService, + private val subscriptionFeed: SubscriptionFeedService, + private val blocked: BlockedService, + private val repository: RssFeedRepository = RssFeedRepository(), + private val throttle: RssFeedThrottle = RssFeedThrottle(), + private val secrets: RssFeedSecret = RssFeedSecret(), + private val clock: () -> Long = System::currentTimeMillis, +) { + suspend fun read(feedId: String, secret: String): RssFeedReadResult { + val config = settings.get() + val baseUrl = config.rssPublicBaseUrl + if (!config.rssEnabled || baseUrl == null) return RssFeedReadResult.NotFound + val stored = repository.find(feedId) ?: return RssFeedReadResult.NotFound + if (!stored.item.enabled || !repository.userEnabled(stored.userId)) return RssFeedReadResult.NotFound + if (!secrets.matches(secret, stored.tokenHash)) return RssFeedReadResult.NotFound + throttle.acquire(feedId, config.rssRateLimitPerMinute)?.let { return RssFeedReadResult.Throttled(it) } + + val scopeChannels = stored.item.channelUrls.takeIf { stored.item.scope == "channels" }?.toSet() + val profile = blocked.profileFor(stored.userId) + val now = clock() + val videos = subscriptionFeed.getCachedAll(stored.userId).orEmpty() + .asSequence() + .filter { scopeChannels == null || ChannelUrlCanonicalizer.canonicalize(it.uploaderUrl) in scopeChannels } + .filter { RssVideoMetadata.serviceId(it) in stored.item.serviceIds } + .filter { RssVideoTypeFilter.includes(stored.item, it, now) } + .filter { profile.allowsVideo(it.url, it.title, it.uploaderUrl, it.uploaderName) } + .take(config.rssMaxItems) + .toList() + val lastModified = RssDocumentRenderer.lastModified(stored.item, videos, now) + val bytes = RssDocumentRenderer.render(stored.item, videos, baseUrl, lastModified) + val etag = "\"${HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes))}\"" + if (stored.item.lastUsedAt == null || now - stored.item.lastUsedAt >= LAST_USED_WRITE_INTERVAL_MS) { + repository.touch(feedId, now) + } + return RssFeedReadResult.Ready(bytes, etag, lastModified, config.rssMinimumPollMinutes * 60) + } + + companion object { + private const val LAST_USED_WRITE_INTERVAL_MS = 60_000L + } +} + +sealed interface RssFeedReadResult { + data class Ready( + val bytes: ByteArray, + val etag: String, + val lastModified: Long, + val maxAgeSeconds: Int, + ) : RssFeedReadResult + + data class Throttled(val retryAfterSeconds: Int) : RssFeedReadResult + data object NotFound : RssFeedReadResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt b/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt new file mode 100644 index 00000000..87e81317 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssFeedThrottle.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.services + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +internal class RssFeedThrottle(private val clock: () -> Long = System::currentTimeMillis) { + private val windows = ConcurrentHashMap() + private val acquisitions = AtomicInteger() + + fun acquire(feedId: String, limit: Int): Int? { + val now = clock() + if (acquisitions.incrementAndGet() % CLEANUP_INTERVAL == 0) { + windows.entries.removeIf { now - it.value.startedAt >= RETENTION_MS } + } + val window = windows.compute(feedId) { _, current -> + if (current == null || now - current.startedAt >= WINDOW_MS) Window(now, 1) else current.copy(count = current.count + 1) + }!! + if (window.count <= limit) return null + return ((WINDOW_MS - (now - window.startedAt) + 999L) / 1_000L).coerceAtLeast(1L).toInt() + } + + private data class Window(val startedAt: Long, val count: Int) + + companion object { + private const val WINDOW_MS = 60_000L + private const val RETENTION_MS = WINDOW_MS * 2 + private const val CLEANUP_INTERVAL = 256 + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt new file mode 100644 index 00000000..047eaaab --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoMetadata.kt @@ -0,0 +1,20 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem +import java.net.URI + +internal object RssVideoMetadata { + fun serviceId(video: VideoItem): Int { + val host = runCatching { URI(video.url).host.orEmpty().lowercase() }.getOrDefault("") + return when { + host == "b23.tv" || host == "bilibili.com" || host.endsWith(".bilibili.com") -> 5 + host == "nico.ms" || host == "nicovideo.jp" || host.endsWith(".nicovideo.jp") -> 6 + else -> 0 + } + } + + fun publishedAtMillis(video: VideoItem): Long { + val value = video.publishedAt?.takeIf { it > 0 } ?: video.uploaded.takeIf { it > 0 } ?: 0L + return if (value in 1..9_999_999_999L) value * 1_000L else value + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt new file mode 100644 index 00000000..1e4e3dc6 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -0,0 +1,16 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem + +internal object RssVideoTypeFilter { + fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { + video.isLive -> feed.includeLive + isUpcoming(video, now) -> feed.includeUpcoming + video.isShortFormContent -> feed.includeShorts + else -> feed.includeVideos + } + + private fun isUpcoming(video: VideoItem, now: Long): Boolean = + !video.isPostLive && video.duration < 0 && RssVideoMetadata.publishedAtMillis(video) > now +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 48a09fc2..602c634e 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -80,6 +80,9 @@ class SubscriptionFeedService( return snapshot.page(page * limit, limit, isRefreshing(userId)) } + internal suspend fun getCachedAll(userId: String): List? = + store.current(userId)?.videos + suspend fun invalidate(userId: String) { runCatching { store.invalidate(userId, UUID.randomUUID().toString()) } .onFailure { logger.warn("subscription_feed event=invalidate_failed user={} error={}", userKey(userId), it.message) } From 8ec0fffd45a0b9ceac18780146cceeabe224ff8f Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:30:57 +0200 Subject: [PATCH 15/22] feat: expose private RSS feed API --- .../dev/typetype/server/ApplicationRoutes.kt | 4 + .../dev/typetype/server/ServiceRegistry.kt | 11 ++ .../typetype/server/routes/AdminRssRoutes.kt | 76 ++++++++++ .../typetype/server/routes/RssFeedRoutes.kt | 141 ++++++++++++++++++ .../typetype/server/routes/UserDataRoutes.kt | 1 + 5 files changed, 233 insertions(+) create mode 100644 src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt diff --git a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt index 44d80b01..41e04025 100644 --- a/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/ApplicationRoutes.kt @@ -3,6 +3,7 @@ package dev.typetype.server import dev.typetype.server.routes.adminBugReportRoutes import dev.typetype.server.routes.adminAllowListRoutes import dev.typetype.server.routes.adminRoutes +import dev.typetype.server.routes.adminRssRoutes import dev.typetype.server.routes.adminIdentityRoutes import dev.typetype.server.routes.adminSessionRoutes import dev.typetype.server.routes.authRoutes @@ -17,6 +18,7 @@ import dev.typetype.server.routes.oidcAuthRoutes import dev.typetype.server.routes.podcastRoutes import dev.typetype.server.routes.publicMetadataRoutes import dev.typetype.server.routes.publicPlaylistRoutes +import dev.typetype.server.routes.rssPublicRoutes import dev.typetype.server.routes.sabrRoutes import dev.typetype.server.routes.searchRoutes import dev.typetype.server.routes.sessionActivityRoutes @@ -66,6 +68,7 @@ internal fun Application.installApplicationRoutes( routing { internalObservabilityRoutes(internalHealthService::check) publicMetadataRoutes(instanceService::getInstance) + rssPublicRoutes(svc.rssFeedReaderService) installStreamRoutes(svc, authService, adminSettingsService) rateLimit(DEARROW_ZONE) { deArrowRoutes(svc.deArrowService) } rateLimit(EXTRACTION_ZONE) { @@ -102,6 +105,7 @@ internal fun Application.installApplicationRoutes( authSessionConfig, ) adminRoutes(authService, userAdminService, passwordResetService, adminSettingsService) + adminRssRoutes(svc.rssFeedManagementService, authService) adminIdentityRoutes(svc.accountIdentityService, authService) adminAllowListRoutes(authService, userAdminService, svc.adminManagedAccessService, svc.adminUserLookupService, svc.allowedChannelsService, svc.allowedPlaylistsService) adminSessionRoutes(authService, activeSessionService) diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index b0cff887..a6ee84ed 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -18,6 +18,8 @@ import dev.typetype.server.services.HomeRecommendationService import dev.typetype.server.services.NotificationsService import dev.typetype.server.services.PlaylistService import dev.typetype.server.services.ProgressService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReaderService import dev.typetype.server.services.PublicHlsManifestTokenService import dev.typetype.server.services.SavedPlaylistService import dev.typetype.server.services.SearchHistoryService @@ -107,6 +109,15 @@ internal class ServiceRegistry( val adminUserLookupService = AdminUserLookupService() val accessControlService = AccessControlService(settingsService, allowedChannelsService, allowedPlaylistsService, adminSettingsService) val blockedService = BlockedService() + val rssFeedManagementService = RssFeedManagementService( + adminSettingsService, + subscriptionsService, + ) + val rssFeedReaderService = RssFeedReaderService( + adminSettingsService, + subscriptionFeedService, + blockedService, + ) val typeTypeBackupService = TypeTypeBackupService( subscriptionsService, historyService, diff --git a/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt new file mode 100644 index 00000000..75f4fc2c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/AdminRssRoutes.kt @@ -0,0 +1,76 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.RssFeedEnabledRequest +import dev.typetype.server.models.RssUserPolicyRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import io.ktor.http.HttpStatusCode +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.put + +fun Route.adminRssRoutes(service: RssFeedManagementService, authService: AuthService) { + get("/admin/rss/feeds") { + call.withRssAdmin(authService) { + val pageRaw = call.request.queryParameters["page"] + val limitRaw = call.request.queryParameters["limit"] + if ( + (pageRaw != null && pageRaw.toIntOrNull() == null) || + (limitRaw != null && limitRaw.toIntOrNull() == null) + ) { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid pagination")) + } + val page = pageRaw?.toInt() ?: 1 + val limit = limitRaw?.toInt() ?: 50 + if (page < 1 || limit !in 1..200) { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid pagination")) + } + call.respondNoStore(service.adminList(page, limit)) + } + } + put("/admin/rss/feeds/{id}/enabled") { + call.withRssAdmin(authService) { + val id = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id")) + val body = runCatching { call.receive() }.getOrElse { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + call.respondNoStore(service.adminSetEnabled(id, body.enabled)) + } + } + delete("/admin/rss/feeds/{id}") { + call.withRssAdmin(authService) { + val id = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id")) + service.adminDelete(id) + call.respond(HttpStatusCode.NoContent) + } + } + put("/admin/rss/users/{id}/enabled") { + call.withRssAdmin(authService) { + val userId = call.parameters["id"] + ?: return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing user id")) + val body = runCatching { call.receive() }.getOrElse { + return@withRssAdmin call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + service.adminSetUserEnabled(userId, body.enabled) + call.respond(HttpStatusCode.NoContent) + } + } +} + +private suspend inline fun io.ktor.server.application.ApplicationCall.withRssAdmin( + authService: AuthService, + crossinline block: suspend () -> Unit, +) { + try { + withAdminAuth(authService) { block() } + } catch (error: RssFeedException) { + respondRssError(error) + } +} diff --git a/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt new file mode 100644 index 00000000..b36b0faa --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/RssFeedRoutes.kt @@ -0,0 +1,141 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.RssFeedEnabledRequest +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReadResult +import dev.typetype.server.services.RssFeedReaderService +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.response.respondBytes +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit + +fun Route.rssFeedRoutes(service: RssFeedManagementService, authService: AuthService) { + get("/rss/feeds") { + call.withRssUser(authService) { userId -> call.respondNoStore(service.list(userId)) } + } + post("/rss/feeds") { + call.withRssUser(authService) { userId -> + val body = call.rssBody() ?: return@withRssUser + call.response.headers.append(HttpHeaders.CacheControl, "no-store") + call.respond(HttpStatusCode.Created, service.create(userId, body)) + } + } + put("/rss/feeds/{id}") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + val body = call.rssBody() ?: return@withRssUser + call.respondNoStore(service.update(userId, id, body)) + } + } + put("/rss/feeds/{id}/enabled") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + val body = call.rssBody() ?: return@withRssUser + call.respondNoStore(service.setEnabled(userId, id, body.enabled)) + } + } + post("/rss/feeds/{id}/regenerate") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + call.respondNoStore(service.regenerate(userId, id)) + } + } + delete("/rss/feeds/{id}") { + call.withRssUser(authService) { userId -> + val id = call.rssFeedId() ?: return@withRssUser + service.delete(userId, id) + call.respond(HttpStatusCode.NoContent) + } + } +} + +fun Route.rssPublicRoutes(service: RssFeedReaderService) { + get("/rss/feeds/{file}") { + val feedId = call.parameters["file"]?.takeIf { it.endsWith(".xml") }?.removeSuffix(".xml") + ?: return@get call.respond(HttpStatusCode.NotFound) + val secret = call.request.queryParameters["token"]?.takeIf(String::isNotBlank) + ?: return@get call.respond(HttpStatusCode.NotFound) + when (val result = service.read(feedId, secret)) { + RssFeedReadResult.NotFound -> call.respond(HttpStatusCode.NotFound) + is RssFeedReadResult.Throttled -> { + call.response.headers.append(HttpHeaders.RetryAfter, result.retryAfterSeconds.toString()) + call.respond(HttpStatusCode.TooManyRequests, ErrorResponse("Too many RSS requests", "rss_rate_limited")) + } + is RssFeedReadResult.Ready -> call.respondRss(result) + } + } +} + +internal suspend fun ApplicationCall.respondRssError(error: RssFeedException) { + val status = when (error.code) { + "rss_feed_not_found", "rss_user_not_found" -> HttpStatusCode.NotFound + "rss_disabled", "rss_user_disabled" -> HttpStatusCode.Forbidden + "rss_feed_limit_reached" -> HttpStatusCode.Conflict + else -> HttpStatusCode.BadRequest + } + respond(status, ErrorResponse(error.message ?: "Invalid RSS request", error.code)) +} + +private suspend inline fun ApplicationCall.withRssUser( + authService: AuthService, + crossinline block: suspend (String) -> Unit, +) { + try { + withJwtAuth(authService) { userId -> + if (userId.startsWith("guest:")) { + return@withJwtAuth respond(HttpStatusCode.Forbidden, ErrorResponse("Guest users cannot manage RSS feeds")) + } + block(userId) + } + } catch (error: RssFeedException) { + respondRssError(error) + } +} + +private suspend inline fun ApplicationCall.rssBody(): T? = runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body", "rss_invalid_body")) + null +} + +private suspend fun ApplicationCall.rssFeedId(): String? = parameters["id"] ?: run { + respond(HttpStatusCode.BadRequest, ErrorResponse("Missing RSS feed id", "rss_missing_feed_id")) + null +} + +private suspend fun ApplicationCall.respondRss(result: RssFeedReadResult.Ready) { + response.headers.append(HttpHeaders.ETag, result.etag) + response.headers.append(HttpHeaders.LastModified, RFC_1123.format(Instant.ofEpochMilli(result.lastModified))) + response.headers.append(HttpHeaders.CacheControl, "private, max-age=${result.maxAgeSeconds}, must-revalidate") + val ifNoneMatch = request.headers[HttpHeaders.IfNoneMatch] + val unchanged = if (ifNoneMatch != null) { + etagMatches(ifNoneMatch, result.etag) + } else { + request.headers[HttpHeaders.IfModifiedSince]?.let(::parseHttpDate)?.let { since -> + !Instant.ofEpochMilli(result.lastModified).truncatedTo(ChronoUnit.SECONDS).isAfter(since) + } == true + } + if (unchanged) return respond(HttpStatusCode.NotModified) + respondBytes(result.bytes, ContentType.parse("application/rss+xml; charset=utf-8")) +} + +private fun parseHttpDate(value: String): Instant? = runCatching { Instant.from(RFC_1123.parse(value)) }.getOrNull() +private fun etagMatches(header: String, etag: String): Boolean = header.split(',').any { candidate -> + candidate.trim().let { it == "*" || it.removePrefix("W/") == etag.removePrefix("W/") } +} +private val RFC_1123 = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC) diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index eab6f335..308248ac 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -20,6 +20,7 @@ internal fun Route.userDataRoutes( subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) subscriptionFeedRoutes(svc.subscriptionFeedService, authService) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) + rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) savedPlaylistRoutes(svc.savedPlaylistService, svc.publicPlaylistService, authService) watchLaterRoutes(svc.watchLaterService, authService, svc.videoMetadataRepairService) From 9ffa81525ec07682530a7976098a7fa655ad5bd5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:01 +0200 Subject: [PATCH 16/22] docs: document private RSS feed API --- openapi.yaml | 14 +++ openapi/components/access-control.yaml | 6 ++ openapi/components/instance.yaml | 12 +++ openapi/components/rss.yaml | 67 ++++++++++++++ openapi/paths/rss-admin.yaml | 69 ++++++++++++++ openapi/paths/rss.yaml | 121 +++++++++++++++++++++++++ 6 files changed, 289 insertions(+) create mode 100644 openapi/components/rss.yaml create mode 100644 openapi/paths/rss-admin.yaml create mode 100644 openapi/paths/rss.yaml diff --git a/openapi.yaml b/openapi.yaml index 00121b5c..f551cc64 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -16,6 +16,7 @@ tags: - name: downloader - name: youtube-session - name: user-data + - name: rss paths: /health: { $ref: ./openapi/paths/health.yaml#/Health } /instance: { $ref: ./openapi/paths/metadata.yaml#/Instance } @@ -43,6 +44,11 @@ paths: /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } + /rss/feeds: { $ref: ./openapi/paths/rss.yaml#/RssFeeds } + /rss/feeds/{id}: { $ref: ./openapi/paths/rss.yaml#/RssFeed } + /rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss.yaml#/RssFeedEnabled } + /rss/feeds/{id}/regenerate: { $ref: ./openapi/paths/rss.yaml#/RssFeedRegenerate } + /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore } @@ -51,6 +57,10 @@ paths: /allowed/channels: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannels } /allowed/channels/{channelUrl}: { $ref: ./openapi/paths/access-control.yaml#/AllowedChannel } /admin/settings: { $ref: ./openapi/paths/access-control.yaml#/AdminSettings } + /admin/rss/feeds: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeeds } + /admin/rss/feeds/{id}/enabled: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeedEnabled } + /admin/rss/feeds/{id}: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssFeed } + /admin/rss/users/{id}/enabled: { $ref: ./openapi/paths/rss-admin.yaml#/AdminRssUserEnabled } /admin/users: { $ref: ./openapi/paths/admin-users.yaml#/AdminUsers } /admin/users/{id}/access-mode: { $ref: ./openapi/paths/admin-users.yaml#/AdminUserAccessMode } /admin/users/managed-access: { $ref: ./openapi/paths/admin-managed-access.yaml#/AdminManagedAccessUsers } @@ -138,6 +148,10 @@ components: SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } + RssFeedRequest: { $ref: ./openapi/components/rss.yaml#/RssFeedRequest } + RssFeedItem: { $ref: ./openapi/components/rss.yaml#/RssFeedItem } + RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem } + AdminRssFeedsPage: { $ref: ./openapi/components/rss.yaml#/AdminRssFeedsPage } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem } TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary } diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index f3e34bab..d8ad0e10 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -37,6 +37,12 @@ SettingsItem: type: string enum: [unrestricted, allow_list] default: unrestricted + rssEnabled: { type: boolean, default: false } + rssPublicBaseUrl: { type: string, format: uri, nullable: true } + rssMaxFeedsPerUser: { type: integer, minimum: 1, maximum: 100, default: 10 } + rssMaxItems: { type: integer, minimum: 1, maximum: 200, default: 50 } + rssMinimumPollMinutes: { type: integer, minimum: 1, maximum: 1440, default: 5 } + rssRateLimitPerMinute: { type: integer, minimum: 1, maximum: 600, default: 30 } AllowedChannelItem: type: object required: [url] diff --git a/openapi/components/instance.yaml b/openapi/components/instance.yaml index 385bb0fc..a0629e47 100644 --- a/openapi/components/instance.yaml +++ b/openapi/components/instance.yaml @@ -20,6 +20,7 @@ InstanceResponse: - oidcAutoRedirect - youtubeRemoteLoginEnabled - youtubeRemoteLoginReady + - rss properties: name: { type: string, example: TypeType } tagline: { type: string, nullable: true } @@ -49,3 +50,14 @@ InstanceResponse: type: string nullable: true enum: [disabled, not_configured, token_unreachable] + rss: + $ref: '#/RssInstanceCapability' +RssInstanceCapability: + type: object + required: [enabled, maxFeedsPerUser, maxItems, minimumPollMinutes, rateLimitPerMinute] + properties: + enabled: { type: boolean } + maxFeedsPerUser: { type: integer } + maxItems: { type: integer } + minimumPollMinutes: { type: integer } + rateLimitPerMinute: { type: integer } diff --git a/openapi/components/rss.yaml b/openapi/components/rss.yaml new file mode 100644 index 00000000..cbd66a5d --- /dev/null +++ b/openapi/components/rss.yaml @@ -0,0 +1,67 @@ +RssFeedRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 100 } + scope: { type: string, enum: [all, channels], default: all } + channelUrls: + type: array + maxItems: 100 + items: { type: string, format: uri } + serviceIds: + type: array + minItems: 1 + uniqueItems: true + default: [0, 5, 6] + items: { type: integer, enum: [0, 5, 6] } + includeVideos: { type: boolean, default: true } + includeShorts: { type: boolean, default: true } + includeLive: { type: boolean, default: true } + includeUpcoming: { type: boolean, default: true } +RssFeedItem: + allOf: + - $ref: '#/RssFeedRequest' + - type: object + required: [id, enabled, createdAt, updatedAt, lastUsedAt] + properties: + id: { type: string } + enabled: { type: boolean } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } + lastUsedAt: { type: integer, format: int64, nullable: true } +RssFeedSecretItem: + type: object + required: [feed, feedUrl] + properties: + feed: { $ref: '#/RssFeedItem' } + feedUrl: + type: string + format: uri + description: Returned only after creation or secret regeneration. +RssFeedEnabledRequest: + type: object + required: [enabled] + properties: + enabled: { type: boolean } +RssUserPolicyRequest: + $ref: '#/RssFeedEnabledRequest' +AdminRssFeedItem: + type: object + required: [feed, userId, userName, userEmail, userRssEnabled, userSuspended] + properties: + feed: { $ref: '#/RssFeedItem' } + userId: { type: string } + userName: { type: string } + userEmail: { type: string, format: email } + userRssEnabled: { type: boolean } + userSuspended: { type: boolean } +AdminRssFeedsPage: + type: object + required: [items, page, limit, total] + properties: + items: + type: array + items: { $ref: '#/AdminRssFeedItem' } + page: { type: integer } + limit: { type: integer } + total: { type: integer, format: int64 } diff --git a/openapi/paths/rss-admin.yaml b/openapi/paths/rss-admin.yaml new file mode 100644 index 00000000..0c431b8f --- /dev/null +++ b/openapi/paths/rss-admin.yaml @@ -0,0 +1,69 @@ +AdminRssFeeds: + get: + tags: [rss] + summary: List private RSS feeds across the instance + security: [{ bearerAuth: [] }] + parameters: + - { name: page, in: query, schema: { type: integer, minimum: 1, default: 1 } } + - { name: limit, in: query, schema: { type: integer, minimum: 1, maximum: 200, default: 50 } } + responses: + '200': + description: Paginated RSS feeds and owners + content: + application/json: + schema: { $ref: ../components/rss.yaml#/AdminRssFeedsPage } + '400': { description: Invalid pagination } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } +AdminRssFeedEnabled: + put: + tags: [rss] + summary: Enable or disable one RSS feed as an admin + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedEnabledRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Feed not found } +AdminRssFeed: + delete: + tags: [rss] + summary: Revoke and delete one RSS feed as an admin + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '204': { description: Deleted } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Feed not found } +AdminRssUserEnabled: + put: + tags: [rss] + summary: Enable or disable RSS for one account + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssUserPolicyRequest } + responses: + '204': { description: Account RSS policy updated } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: Admin role required } + '404': { description: Account not found } diff --git a/openapi/paths/rss.yaml b/openapi/paths/rss.yaml new file mode 100644 index 00000000..bd8e610c --- /dev/null +++ b/openapi/paths/rss.yaml @@ -0,0 +1,121 @@ +RssFeeds: + get: + tags: [rss] + summary: List private RSS feeds for the current account + security: [{ bearerAuth: [] }] + responses: + '200': + description: RSS feeds without their secrets + content: + application/json: + schema: { type: array, items: { $ref: ../components/rss.yaml#/RssFeedItem } } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + post: + tags: [rss] + summary: Create a private RSS feed + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedRequest } + responses: + '201': + description: Created feed and its one-time private URL + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedSecretItem } + '400': { description: Invalid scope or filters } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '409': { description: Account feed limit reached } +RssFeed: + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + put: + tags: [rss] + summary: Replace a private RSS feed configuration + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid scope or filters } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } + delete: + tags: [rss] + summary: Delete a private RSS feed + security: [{ bearerAuth: [] }] + responses: + '204': { description: Deleted } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedEnabled: + put: + tags: [rss] + summary: Enable or disable a private RSS feed + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedEnabledRequest } + responses: + '200': + description: Updated feed + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedItem } + '400': { description: Invalid request body } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedRegenerate: + post: + tags: [rss] + summary: Revoke the current secret and issue a new private URL + security: [{ bearerAuth: [] }] + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + responses: + '200': + description: Feed and its replacement one-time private URL + content: + application/json: + schema: { $ref: ../components/rss.yaml#/RssFeedSecretItem } + '401': { description: Missing or invalid token } + '403': { description: RSS is disabled globally or for this account } + '404': { description: Feed not found for this account } +RssFeedDocument: + get: + tags: [rss] + summary: Read a private RSS 2.0 document + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + - { name: token, in: query, required: true, schema: { type: string } } + responses: + '200': + description: RSS document + headers: + ETag: { schema: { type: string } } + Last-Modified: { schema: { type: string } } + Cache-Control: { schema: { type: string } } + content: + application/rss+xml: + schema: { type: string } + '304': { description: Document unchanged } + '404': { description: Invalid, disabled, or revoked feed } + '429': { description: Feed request limit exceeded } From 33717e21e4e49c0a9a0ccf7def858a0355279dd9 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 17/22] test: cover RSS feed management --- .../server/RssFeedManagementServiceTest.kt | 155 ++++++++++++++++++ .../dev/typetype/server/RssFeedSecretTest.kt | 23 +++ 2 files changed, 178 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt new file mode 100644 index 00000000..b6f82cc4 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedManagementServiceTest.kt @@ -0,0 +1,155 @@ +package dev.typetype.server + +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.RssFeedException +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction + +class RssFeedManagementServiceTest { + private val settings = AdminSettingsService() + private val subscriptions = SubscriptionsService() + private val service = RssFeedManagementService(settings, subscriptions) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + } + + @Test + fun `selected channels must be owned subscriptions and feeds stay isolated`() = runTest { + enableRss() + insertUser("user-a") + insertUser("user-b") + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + subscriptions.add("user-b", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@b", "B")) + + val created = service.create( + "user-a", + RssFeedRequest(name = "A only", scope = "channels", channelUrls = listOf("https://youtube.com/@a/")), + ) + + assertTrue(created.feedUrl.startsWith("https://video.example/api/rss/feeds/")) + assertEquals(listOf("https://youtube.com/@a"), created.feed.channelUrls) + assertEquals(listOf(created.feed), service.list("user-a")) + assertTrue(service.list("user-b").isEmpty()) + val error = runCatching { + service.create( + "user-b", + RssFeedRequest(name = "Not mine", scope = "channels", channelUrls = listOf("https://youtube.com/@a")), + ) + }.exceptionOrNull() as RssFeedException + assertEquals("rss_channel_not_subscribed", error.code) + } + + @Test + fun `accounts cannot mutate feeds they do not own`() = runTest { + enableRss() + insertUser("user-a") + insertUser("user-b") + val created = service.create("user-a", RssFeedRequest(name = "Private feed")) + + val attempts = listOf Unit>( + { service.update("user-b", created.feed.id, RssFeedRequest(name = "Changed")) }, + { service.setEnabled("user-b", created.feed.id, false) }, + { service.regenerate("user-b", created.feed.id) }, + { service.delete("user-b", created.feed.id) }, + ) + + attempts.forEach { attempt -> + val error = runCatching { attempt() }.exceptionOrNull() as RssFeedException + assertEquals("rss_feed_not_found", error.code) + } + assertEquals(listOf(created.feed), service.list("user-a")) + } + + @Test + fun `disabled account retains feeds but cannot manage them`() = runTest { + enableRss() + insertUser("user-a") + service.create("user-a", RssFeedRequest(name = "All")) + service.adminSetUserEnabled("user-a", false) + + val error = runCatching { service.list("user-a") }.exceptionOrNull() as RssFeedException + assertEquals("rss_user_disabled", error.code) + assertEquals(1L, service.adminList(1, 20).total) + } + + @Test + fun `global disable retains feed configuration`() = runTest { + enableRss() + insertUser("user-a") + val created = service.create("user-a", RssFeedRequest(name = "All")) + settings.upsert(AdminSettingsItem(rssEnabled = false, rssPublicBaseUrl = "https://video.example")) + + val error = runCatching { service.list("user-a") }.exceptionOrNull() as RssFeedException + assertEquals("rss_disabled", error.code) + assertEquals(created.feed, service.adminList(1, 20).items.single().feed) + + enableRss() + assertEquals(listOf(created.feed), service.list("user-a")) + } + + @Test + fun `admin cannot create an RSS policy for an unknown account`() = runTest { + val error = runCatching { service.adminSetUserEnabled("missing", false) } + .exceptionOrNull() as RssFeedException + + assertEquals("rss_user_not_found", error.code) + } + + @Test + fun `concurrent creation cannot exceed the account feed limit`() = runTest { + settings.upsert( + AdminSettingsItem( + rssEnabled = true, + rssPublicBaseUrl = "https://video.example", + rssMaxFeedsPerUser = 1, + ), + ) + insertUser("user-a") + + val attempts = List(4) { index -> + async { runCatching { service.create("user-a", RssFeedRequest(name = "Feed $index")) } } + }.awaitAll() + + assertEquals(1, attempts.count(Result<*>::isSuccess)) + assertTrue(attempts.filter(Result<*>::isFailure).all { + (it.exceptionOrNull() as RssFeedException).code == "rss_feed_limit_reached" + }) + } + + private suspend fun enableRss() { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + } + + private fun insertUser(id: String) = transaction { + UsersTable.insert { + it[UsersTable.id] = id + it[email] = "$id@test.local" + it[passwordHash] = "hash" + it[name] = id + it[role] = "user" + it[createdAt] = 1L + it[updatedAt] = 1L + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt new file mode 100644 index 00000000..c85db17e --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedSecretTest.kt @@ -0,0 +1,23 @@ +package dev.typetype.server + +import dev.typetype.server.services.RssFeedSecret +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssFeedSecretTest { + private val secrets = RssFeedSecret() + + @Test + fun `secret is random and only its hash is comparable`() { + val first = secrets.create() + val second = secrets.create() + val hash = secrets.hash(first) + + assertNotEquals(first, second) + assertNotEquals(first, hash) + assertTrue(secrets.matches(first, hash)) + assertFalse(secrets.matches(second, hash)) + } +} From 7a520f6b62bd04f363b1d2af5c786a3afee59107 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 18/22] test: cover private RSS delivery --- .../server/RssFeedReaderRoutesTest.kt | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt diff --git a/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt b/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt new file mode 100644 index 00000000..2c109240 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssFeedReaderRoutesTest.kt @@ -0,0 +1,221 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.ChannelPlaylistsResponse +import dev.typetype.server.models.ChannelResponse +import dev.typetype.server.models.ExtractionResult +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.routes.rssPublicRoutes +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.BlockedService +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.RssFeedReaderService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.testApplication +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import java.net.URI +import java.net.URLEncoder +import java.nio.charset.StandardCharsets + +class RssFeedReaderRoutesTest { + private val settings = AdminSettingsService() + private val subscriptions = SubscriptionsService() + private val feed = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + private val blocked = BlockedService() + private val management = RssFeedManagementService(settings, subscriptions) + private val reader = RssFeedReaderService(settings, feed, blocked) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + insertUser() + } + + @Test + fun `private feed supports conditional cache and immediate secret revocation`() = testApplication { + val created = createFeed() + val uri = URI(created.feedUrl) + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + val first = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + assertEquals(HttpStatusCode.OK, first.status) + assertEquals("application/rss+xml; charset=utf-8", first.headers[HttpHeaders.ContentType]) + assertEquals("private, max-age=300, must-revalidate", first.headers[HttpHeaders.CacheControl]) + assertTrue(first.bodyAsText().contains("")) + assertTrue(first.bodyAsText().contains("https://video.example/watch?v=")) + val etag = first.headers[HttpHeaders.ETag]!! + val lastModified = first.headers[HttpHeaders.LastModified]!! + + val cached = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) { + headers.append(HttpHeaders.IfNoneMatch, "W/$etag") + } + assertEquals(HttpStatusCode.NotModified, cached.status) + + val changed = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) { + headers.append(HttpHeaders.IfNoneMatch, "\"different\"") + headers.append(HttpHeaders.IfModifiedSince, lastModified) + } + assertEquals(HttpStatusCode.OK, changed.status) + + val regenerated = management.regenerate("user-a", created.feed.id) + assertEquals(HttpStatusCode.NotFound, client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery).status) + assertFalse(regenerated.feedUrl.endsWith(uri.rawQuery)) + } + + @Test + fun `blocked videos channels and keywords are excluded from RSS output`() = testApplication { + val created = createFeed() + val uri = URI(created.feedUrl) + val path = uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + blocked.addVideo("user-a", "https://youtube.com/@a/video") + assertTrue(blocked.profileFor("user-a").blocksVideo("https://youtube.com/@a/video")) + val blockedVideoUrl = URLEncoder.encode("https://youtube.com/@a/video", StandardCharsets.UTF_8) + assertFalse(client.get(path).bodyAsText().contains(blockedVideoUrl)) + blocked.deleteVideo("user-a", "https://youtube.com/@a/video", "user") + + blocked.addChannel("user-a", "https://youtube.com/@a") + assertTrue(blocked.profileFor("user-a").blocksChannel("https://youtube.com/@a", "A")) + assertFalse(client.get(path).bodyAsText().contains("")) + blocked.deleteChannel("user-a", "https://youtube.com/@a", "user") + + blocked.addKeyword("user-a", "video") + assertFalse(client.get(path).bodyAsText().contains("")) + } + + @Test + fun `RSS reads an existing snapshot without starting extraction`() = testApplication { + val channel = CountingChannelService() + val snapshotOnlyFeed = SubscriptionFeedService(subscriptions, channel, FakeCacheService()) + val snapshotOnlyReader = RssFeedReaderService(settings, snapshotOnlyFeed, blocked) + val created = createUnprimedFeed() + val uri = URI(created.feedUrl) + application { routing { rssPublicRoutes(snapshotOnlyReader) } } + + val response = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + + assertEquals(HttpStatusCode.OK, response.status) + assertFalse(response.bodyAsText().contains("")) + assertEquals(0, channel.calls) + } + + @Test + fun `RSS enforces configured item and request limits`() = testApplication { + settings.upsert( + AdminSettingsItem( + rssEnabled = true, + rssPublicBaseUrl = "https://video.example", + rssMaxItems = 1, + rssRateLimitPerMinute = 1, + ), + ) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@b", "B")) + val created = management.create("user-a", RssFeedRequest(name = "Limited feed")) + feed.getAll("user-a") + feed.awaitRefresh("user-a") + val uri = URI(created.feedUrl) + val path = uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery + application { + install(ContentNegotiation) { json() } + routing { rssPublicRoutes(reader) } + } + + val first = client.get(path) + assertEquals(1, "".toRegex().findAll(first.bodyAsText()).count()) + val throttled = client.get(path) + assertEquals(HttpStatusCode.TooManyRequests, throttled.status) + assertTrue(throttled.headers[HttpHeaders.RetryAfter]?.toIntOrNull() in 1..60) + } + + @Test + fun `RSS rejects feeds owned by suspended accounts`() = testApplication { + val created = createFeed() + transaction { + UsersTable.update({ UsersTable.id eq "user-a" }) { it[suspended] = true } + } + val uri = URI(created.feedUrl) + application { routing { rssPublicRoutes(reader) } } + + val response = client.get(uri.rawPath.removePrefix("/api") + "?" + uri.rawQuery) + + assertEquals(HttpStatusCode.NotFound, response.status) + } + + private suspend fun createFeed(): dev.typetype.server.models.RssFeedSecretItem { + val created = createUnprimedFeed() + feed.getAll("user-a") + feed.awaitRefresh("user-a") + return created + } + + private suspend fun createUnprimedFeed(): dev.typetype.server.models.RssFeedSecretItem { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + subscriptions.add("user-a", SubscriptionFeedTestFixtures.subscription("https://youtube.com/@a", "A")) + return management.create("user-a", RssFeedRequest(name = "My feed")) + } + + private fun insertUser() = transaction { + UsersTable.insert { + it[id] = "user-a" + it[email] = "rss-reader@test.local" + it[passwordHash] = "hash" + it[name] = "RSS reader" + it[role] = "user" + it[createdAt] = 1L + it[updatedAt] = 1L + } + } + + private class CountingChannelService : ChannelService { + var calls = 0 + + override suspend fun getChannel( + url: String, + nextpage: String?, + sort: String?, + ): ExtractionResult { + calls += 1 + return ExtractionResult.Success(ChannelResponse("", "", "", "", 0, false, emptyList(), null)) + } + + override suspend fun getPlaylists( + url: String, + nextpage: String?, + ): ExtractionResult = + ExtractionResult.Success(ChannelPlaylistsResponse(emptyList(), null)) + } +} From 50e41619e7803c20566744546c047f8eadcc13af Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 13:31:07 +0200 Subject: [PATCH 19/22] test: cover RSS policy and content filters --- .../dev/typetype/server/AdminRssRoutesTest.kt | 150 ++++++++++++++++++ .../typetype/server/RssVideoTypeFilterTest.kt | 118 ++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt diff --git a/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt b/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt new file mode 100644 index 00000000..600feb54 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/AdminRssRoutesTest.kt @@ -0,0 +1,150 @@ +package dev.typetype.server + +import dev.typetype.server.db.tables.UsersTable +import dev.typetype.server.models.AdminSettingsItem +import dev.typetype.server.models.RssFeedRequest +import dev.typetype.server.routes.adminRssRoutes +import dev.typetype.server.services.AdminSettingsService +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.RssFeedManagementService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.headers +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.http.contentType +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.jetbrains.exposed.v1.jdbc.insert +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import org.jetbrains.exposed.v1.core.eq +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class AdminRssRoutesTest { + private val settings = AdminSettingsService() + private val service = RssFeedManagementService(settings, SubscriptionsService()) + private val auth = AuthService.fixed(ADMIN_ID) + + companion object { + private const val ADMIN_ID = "rss-admin" + private const val USER_ID = "rss-user" + + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + insertUser(ADMIN_ID, "admin") + insertUser(USER_ID, "user") + } + + @Test + fun `admin can inspect and revoke a feed without receiving its secret`() = withApp { + enableRss() + val created = service.create(USER_ID, RssFeedRequest(name = "Private feed")) + + val listed = client.get("/admin/rss/feeds") { authorize() } + assertEquals(HttpStatusCode.OK, listed.status) + val body = listed.bodyAsText() + assertTrue(body.contains("user@test.local")) + assertTrue(body.contains(created.feed.id)) + assertFalse(body.contains("feedUrl")) + assertFalse(body.contains("token")) + + val deleted = client.delete("/admin/rss/feeds/${created.feed.id}") { authorize() } + assertEquals(HttpStatusCode.NoContent, deleted.status) + assertEquals(0L, service.adminList(1, 20).total) + } + + @Test + fun `admin account policy is retained and unknown accounts return 404`() = withApp { + enableRss() + service.create(USER_ID, RssFeedRequest(name = "Private feed")) + + val disabled = client.put("/admin/rss/users/$USER_ID/enabled") { + authorize() + contentType(ContentType.Application.Json) + setBody("""{"enabled":false}""") + } + assertEquals(HttpStatusCode.NoContent, disabled.status) + val item = service.adminList(1, 20).items.single() + assertFalse(item.userRssEnabled) + assertTrue(item.feed.enabled) + + val missing = client.put("/admin/rss/users/missing/enabled") { + authorize() + contentType(ContentType.Application.Json) + setBody("""{"enabled":false}""") + } + assertEquals(HttpStatusCode.NotFound, missing.status) + assertTrue(missing.bodyAsText().contains("rss_user_not_found")) + } + + @Test + fun `admin inventory reports suspended owners`() = withApp { + enableRss() + service.create(USER_ID, RssFeedRequest(name = "Private feed")) + transaction { + UsersTable.update({ UsersTable.id eq USER_ID }) { it[suspended] = true } + } + + val response = client.get("/admin/rss/feeds") { authorize() } + + assertEquals(HttpStatusCode.OK, response.status) + assertTrue(response.bodyAsText().contains("\"userSuspended\":true")) + } + + @Test + fun `admin inventory rejects malformed pagination`() = withApp { + val response = client.get("/admin/rss/feeds?page=invalid") { authorize() } + + assertEquals(HttpStatusCode.BadRequest, response.status) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json(Json { encodeDefaults = true }) } + routing { adminRssRoutes(service, auth) } + } + block() + } + + private suspend fun enableRss() { + settings.upsert(AdminSettingsItem(rssEnabled = true, rssPublicBaseUrl = "https://video.example")) + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + headers.append(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun insertUser(id: String, role: String) = transaction { + UsersTable.insert { + it[UsersTable.id] = id + it[email] = role + "@test.local" + it[passwordHash] = "hash" + it[name] = role + it[UsersTable.role] = role + it[createdAt] = 1L + it[updatedAt] = 1L + } + } +} diff --git a/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt b/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt new file mode 100644 index 00000000..b14e089b --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/RssVideoTypeFilterTest.kt @@ -0,0 +1,118 @@ +package dev.typetype.server + +import dev.typetype.server.models.RssFeedItem +import dev.typetype.server.models.VideoItem +import dev.typetype.server.services.RssVideoMetadata +import dev.typetype.server.services.RssVideoTypeFilter +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class RssVideoTypeFilterTest { + @Test + fun `separates regular shorts live and upcoming content`() { + assertTrue(RssVideoTypeFilter.includes(feed(includeVideos = true), video(), NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeVideos = false), video(), NOW)) + assertTrue(RssVideoTypeFilter.includes(feed(includeShorts = true), video(short = true), NOW)) + assertTrue(RssVideoTypeFilter.includes(feed(includeLive = true), video(live = true), NOW)) + assertTrue( + RssVideoTypeFilter.includes( + feed(includeUpcoming = true), + video(duration = -1, publishedAt = NOW + 60_000), + NOW, + ), + ) + } + + @Test + fun `does not treat past unknown-duration or post-live videos as upcoming`() { + val upcomingOnly = feed(includeUpcoming = true) + assertFalse( + RssVideoTypeFilter.includes( + upcomingOnly, + video(duration = -1, publishedAt = NOW - 60_000), + NOW, + ), + ) + assertFalse( + RssVideoTypeFilter.includes( + upcomingOnly, + video(duration = -1, publishedAt = NOW + 60_000, postLive = true), + NOW, + ), + ) + } + + @Test + fun `live and upcoming state takes priority over short form`() { + val liveShort = video(short = true, live = true) + assertTrue(RssVideoTypeFilter.includes(feed(includeLive = true), liveShort, NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeShorts = true), liveShort, NOW)) + + val upcomingShort = video(short = true, duration = -1, publishedAt = NOW + 60_000) + assertTrue(RssVideoTypeFilter.includes(feed(includeUpcoming = true), upcomingShort, NOW)) + assertFalse(RssVideoTypeFilter.includes(feed(includeShorts = true), upcomingShort, NOW)) + } + + @Test + fun `detects supported provider from canonical and short hosts`() { + assertEquals(0, RssVideoMetadata.serviceId(video(url = "https://www.youtube.com/watch?v=video"))) + assertEquals(5, RssVideoMetadata.serviceId(video(url = "https://b23.tv/video"))) + assertEquals(5, RssVideoMetadata.serviceId(video(url = "https://www.bilibili.com/video/BV1"))) + assertEquals(6, RssVideoMetadata.serviceId(video(url = "https://nico.ms/sm1"))) + assertEquals(6, RssVideoMetadata.serviceId(video(url = "https://www.nicovideo.jp/watch/sm1"))) + } + + private fun feed( + includeVideos: Boolean = false, + includeShorts: Boolean = false, + includeLive: Boolean = false, + includeUpcoming: Boolean = false, + ) = RssFeedItem( + id = "feed", + name = "Feed", + scope = "all", + channelUrls = emptyList(), + serviceIds = listOf(0), + includeVideos = includeVideos, + includeShorts = includeShorts, + includeLive = includeLive, + includeUpcoming = includeUpcoming, + enabled = true, + createdAt = NOW, + updatedAt = NOW, + ) + + private fun video( + duration: Long = 120, + publishedAt: Long = NOW - 60_000, + short: Boolean = false, + live: Boolean = false, + postLive: Boolean = false, + url: String = "https://youtube.com/watch?v=video", + ) = VideoItem( + id = "video", + title = "Video", + url = url, + thumbnailUrl = "", + uploaderName = "Channel", + uploaderUrl = "https://youtube.com/@channel", + uploaderAvatarUrl = "", + duration = duration, + viewCount = 0, + uploadDate = "", + streamType = "video_stream", + isShortFormContent = short, + uploaderVerified = false, + shortDescription = null, + publishedAt = publishedAt, + isLive = live, + isPostLive = postLive, + isLiveContent = live || postLive, + ) + + private companion object { + const val NOW = 1_800_000_000_000L + } +} From abed280dae88af084bd664a6b247a5e42d30e39b Mon Sep 17 00:00:00 2001 From: Priveetee Date: Sun, 9 Aug 2026 19:23:15 +0200 Subject: [PATCH 20/22] feat: hide live streams from subscription feeds --- openapi/components/access-control.yaml | 1 + .../server/db/SettingsSchemaMigrations.kt | 1 + .../server/db/tables/SettingsTable.kt | 1 + .../typetype/server/models/SettingsItem.kt | 1 + .../server/routes/SubscriptionFeedRoutes.kt | 10 +- .../typetype/server/routes/UserDataRoutes.kt | 2 +- .../server/services/RssVideoTypeFilter.kt | 5 +- .../services/SettingsPersistenceMappers.kt | 2 + .../server/services/SettingsService.kt | 5 + .../services/SubscriptionFeedService.kt | 8 +- .../services/SubscriptionFeedSnapshot.kt | 24 ++-- .../server/services/VideoItemSchedule.kt | 8 ++ .../SettingsPrivacyControlsRoutesTest.kt | 4 +- ...ubscriptionFeedLiveVisibilityRoutesTest.kt | 131 ++++++++++++++++++ 14 files changed, 186 insertions(+), 17 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt diff --git a/openapi/components/access-control.yaml b/openapi/components/access-control.yaml index d8ad0e10..900de1fe 100644 --- a/openapi/components/access-control.yaml +++ b/openapi/components/access-control.yaml @@ -33,6 +33,7 @@ SettingsItem: hideRelatedVideos: { type: boolean, default: false } hideComments: { type: boolean, default: false } hideShorts: { type: boolean, default: false } + hideSubscriptionLiveStreams: { type: boolean, default: false } accessMode: type: string enum: [unrestricted, allow_list] diff --git a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt index cb816f2f..45566d0f 100644 --- a/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt +++ b/src/main/kotlin/dev/typetype/server/db/SettingsSchemaMigrations.kt @@ -24,6 +24,7 @@ object SettingsSchemaMigrations { exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_related_videos BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_comments BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_shorts BOOLEAN NOT NULL DEFAULT false") + exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS hide_subscription_live_streams BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS disable_watch_history BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS skip_playlist_autoplay_screen BOOLEAN NOT NULL DEFAULT false") exec("ALTER TABLE settings ADD COLUMN IF NOT EXISTS subscription_sync_interval INTEGER NOT NULL DEFAULT 0") diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt index d4b78c50..4744441f 100644 --- a/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt +++ b/src/main/kotlin/dev/typetype/server/db/tables/SettingsTable.kt @@ -32,6 +32,7 @@ object SettingsTable : Table("settings") { val hideRelatedVideos = bool("hide_related_videos").default(false) val hideComments = bool("hide_comments").default(false) val hideShorts = bool("hide_shorts").default(false) + val hideSubscriptionLiveStreams = bool("hide_subscription_live_streams").default(false) val disableWatchHistory = bool("disable_watch_history").default(false) val deArrowEnabled = bool("dearrow_enabled").default(false) val deArrowTitleMode = text("dearrow_title_mode").default("dearrow") diff --git a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt index e471b347..b6f46a83 100644 --- a/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt +++ b/src/main/kotlin/dev/typetype/server/models/SettingsItem.kt @@ -32,6 +32,7 @@ data class SettingsItem( val hideRelatedVideos: Boolean = false, val hideComments: Boolean = false, val hideShorts: Boolean = false, + val hideSubscriptionLiveStreams: Boolean = false, val disableWatchHistory: Boolean = false, val deArrowEnabled: Boolean = false, val deArrowTitleMode: String = "dearrow", diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 110d257c..07154402 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -5,6 +5,7 @@ import dev.typetype.server.models.SubscriptionFeedPreparingResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SettingsService import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond @@ -13,14 +14,19 @@ import io.ktor.server.routing.get private const val MAX_FEED_PAGE = 10_000 -fun Route.subscriptionFeedRoutes(feedService: SubscriptionFeedService, authService: AuthService) { +fun Route.subscriptionFeedRoutes( + feedService: SubscriptionFeedService, + authService: AuthService, + settingsService: SettingsService? = null, +) { get("/subscriptions/feed") { call.withJwtAuth(authService) { userId -> val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 val cursor = call.request.queryParameters["cursor"] + val hideLiveStreams = settingsService?.hidesSubscriptionLiveStreams(userId) ?: false call.response.headers.append(HttpHeaders.CacheControl, "no-store") - when (val result = feedService.getPage(userId, page, limit, cursor)) { + when (val result = feedService.getPage(userId, page, limit, cursor, hideLiveStreams)) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) is SubscriptionFeedPageResult.Preparing -> { call.response.headers.append(HttpHeaders.RetryAfter, "1") diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index 308248ac..74baab91 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -18,7 +18,7 @@ internal fun Route.userDataRoutes( ) { historyRoutes(svc.historyService, authService, svc.settingsService) subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) - subscriptionFeedRoutes(svc.subscriptionFeedService, authService) + subscriptionFeedRoutes(svc.subscriptionFeedService, authService, svc.settingsService) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) rssFeedRoutes(svc.rssFeedManagementService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) diff --git a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt index 1e4e3dc6..a7919c07 100644 --- a/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt +++ b/src/main/kotlin/dev/typetype/server/services/RssVideoTypeFilter.kt @@ -6,11 +6,8 @@ import dev.typetype.server.models.VideoItem internal object RssVideoTypeFilter { fun includes(feed: RssFeedItem, video: VideoItem, now: Long): Boolean = when { video.isLive -> feed.includeLive - isUpcoming(video, now) -> feed.includeUpcoming + video.isUpcomingAt(now) -> feed.includeUpcoming video.isShortFormContent -> feed.includeShorts else -> feed.includeVideos } - - private fun isUpcoming(video: VideoItem, now: Long): Boolean = - !video.isPostLive && video.duration < 0 && RssVideoMetadata.publishedAtMillis(video) > now } diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt index 451019b7..f40a0237 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsPersistenceMappers.kt @@ -45,6 +45,7 @@ internal fun ResultRow.toSettingsItem(): SettingsItem = SettingsItem( hideRelatedVideos = this[SettingsTable.hideRelatedVideos], hideComments = this[SettingsTable.hideComments], hideShorts = this[SettingsTable.hideShorts], + hideSubscriptionLiveStreams = this[SettingsTable.hideSubscriptionLiveStreams], disableWatchHistory = this[SettingsTable.disableWatchHistory], deArrowEnabled = this[SettingsTable.deArrowEnabled], deArrowTitleMode = this[SettingsTable.deArrowTitleMode], @@ -82,6 +83,7 @@ internal fun UpdateBuilder<*>.writeSettings(settings: SettingsItem) { this[SettingsTable.hideRelatedVideos] = settings.hideRelatedVideos this[SettingsTable.hideComments] = settings.hideComments this[SettingsTable.hideShorts] = settings.hideShorts + this[SettingsTable.hideSubscriptionLiveStreams] = settings.hideSubscriptionLiveStreams this[SettingsTable.disableWatchHistory] = settings.disableWatchHistory this[SettingsTable.deArrowEnabled] = settings.deArrowEnabled this[SettingsTable.deArrowTitleMode] = settings.deArrowTitleMode diff --git a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt index 2e73bbf3..1dffe3c3 100644 --- a/src/main/kotlin/dev/typetype/server/services/SettingsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SettingsService.kt @@ -35,6 +35,11 @@ class SettingsService { ?.get(SettingsTable.disableWatchHistory) ?: false } + suspend fun hidesSubscriptionLiveStreams(userId: String): Boolean = DatabaseFactory.query { + SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull() + ?.get(SettingsTable.hideSubscriptionLiveStreams) ?: false + } + suspend fun getAccessModePolicy(userId: String): AccessModePolicy = DatabaseFactory.query { SettingsTable.selectAll().where { SettingsTable.userId eq userId }.singleOrNull()?.let { AccessModePolicy( diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 602c634e..f641e2ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -36,6 +36,7 @@ class SubscriptionFeedService( page: Int, limit: Int, cursor: String?, + hideLiveStreams: Boolean = false, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -49,6 +50,9 @@ class SubscriptionFeedService( val cursorState = cursor?.let(SubscriptionFeedCursorCodec::decode) if (cursor != null && cursorState == null) return SubscriptionFeedPageResult.InvalidCursor if (cursorState != null && cursorState.limit != limit) return SubscriptionFeedPageResult.InvalidCursor + if (cursorState != null && cursorState.hideLiveStreams != hideLiveStreams) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current @@ -56,7 +60,9 @@ class SubscriptionFeedService( ?: return SubscriptionFeedPageResult.StaleGeneration } val offset = cursorState?.offset ?: page * limit - return SubscriptionFeedPageResult.Ready(snapshot.page(offset, limit, isRefreshing(userId))) + return SubscriptionFeedPageResult.Ready( + snapshot.page(offset, limit, isRefreshing(userId), hideLiveStreams), + ) } suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index b48aa4a8..2f7fd2ec 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -20,13 +20,14 @@ private data class SubscriptionFeedCursor( val generation: Long, val offset: Int, val limit: Int, + val hideLiveStreams: Boolean = false, ) internal object SubscriptionFeedCursorCodec { - fun encode(generation: Long, offset: Int, limit: Int): String { + fun encode(generation: Long, offset: Int, limit: Int, hideLiveStreams: Boolean): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit), + SubscriptionFeedCursor(generation, offset, limit, hideLiveStreams), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -35,7 +36,7 @@ internal object SubscriptionFeedCursorCodec { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } - ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit) } + ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit, it.hideLiveStreams) } }.getOrNull() } @@ -43,22 +44,29 @@ internal data class SubscriptionFeedCursorState( val generation: Long, val offset: Int, val limit: Int, + val hideLiveStreams: Boolean, ) internal fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, + hideLiveStreams: Boolean = false, ): SubscriptionFeedResponse { - val from = offset.coerceAtMost(videos.size) - val to = minOf(from + limit, videos.size) - val nextpage = if (to < videos.size) { - SubscriptionFeedCursorCodec.encode(generation, to, limit) + val visibleVideos = if (hideLiveStreams) { + videos.filterNot { it.isLiveOrUpcomingAt(generatedAt) } + } else { + videos + } + val from = offset.coerceAtMost(visibleVideos.size) + val to = minOf(from + limit, visibleVideos.size) + val nextpage = if (to < visibleVideos.size) { + SubscriptionFeedCursorCodec.encode(generation, to, limit, hideLiveStreams) } else { null } return SubscriptionFeedResponse( - videos = videos.subList(from, to), + videos = visibleVideos.subList(from, to), nextpage = nextpage, generation = generation, generatedAt = generatedAt, diff --git a/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt new file mode 100644 index 00000000..977b3cd9 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/VideoItemSchedule.kt @@ -0,0 +1,8 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.VideoItem + +internal fun VideoItem.isUpcomingAt(now: Long): Boolean = + !isPostLive && duration < 0 && RssVideoMetadata.publishedAtMillis(this) > now + +internal fun VideoItem.isLiveOrUpcomingAt(now: Long): Boolean = isLive || isUpcomingAt(now) diff --git a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt index 04ca28e9..cf6891bd 100644 --- a/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SettingsPrivacyControlsRoutesTest.kt @@ -63,6 +63,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideRelatedVideos\":false", "\"hideComments\":false", "\"hideShorts\":false", + "\"hideSubscriptionLiveStreams\":false", ), ) } @@ -87,6 +88,7 @@ class SettingsPrivacyControlsRoutesTest { "\"hideRelatedVideos\":true", "\"hideComments\":true", "\"hideShorts\":true", + "\"hideSubscriptionLiveStreams\":true", ), ) } @@ -106,6 +108,6 @@ class SettingsPrivacyControlsRoutesTest { values.forEach { assertTrue(body.contains(it)) } private fun settingsBody(sponsorBlockMode: String = "mark_only"): String = """ - {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true} + {"defaultService":0,"defaultQuality":"1080p","autoplay":true,"volume":1.0,"muted":false,"sponsorBlockMode":"$sponsorBlockMode","hideHomeRecommendations":true,"hideRelatedVideos":true,"hideComments":true,"hideShorts":true,"hideSubscriptionLiveStreams":true} """.trimIndent() } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt new file mode 100644 index 00000000..c92392f2 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedLiveVisibilityRoutesTest.kt @@ -0,0 +1,131 @@ +package dev.typetype.server + +import dev.typetype.server.SubscriptionFeedTestFixtures.channel +import dev.typetype.server.SubscriptionFeedTestFixtures.subscription +import dev.typetype.server.SubscriptionFeedTestFixtures.video +import dev.typetype.server.models.SettingsItem +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.routes.subscriptionFeedRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.ChannelService +import dev.typetype.server.services.SettingsService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionFeedLiveVisibilityRoutesTest { + private val auth = AuthService.fixed(TEST_USER_ID) + private val settingsService = SettingsService() + private lateinit var subscriptionsService: SubscriptionsService + private lateinit var feedService: SubscriptionFeedService + + companion object { @BeforeAll @JvmStatic fun initDb() = TestDatabase.setup() } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + subscriptionsService = SubscriptionsService() + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_500L, url = "https://youtube.com/watch?v=scheduled").copy( + duration = -1L, + publishedAt = System.currentTimeMillis() + 86_400_000L, + ), + video(3_000L, url = "https://youtube.com/watch?v=normal-1"), + video(2_000L, url = "https://youtube.com/watch?v=normal-2"), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + } + + @Test + fun `account setting hides live streams before pagination`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val first = readPage(requestFeed(limit = 1)) + val cursor = requireNotNull(first.nextpage) + val second = readPage(requestFeed(limit = 1, cursor = cursor)) + + assertEquals(listOf("normal-1"), first.videos.map { it.url.substringAfter("v=") }) + assertEquals(listOf("normal-2"), second.videos.map { it.url.substringAfter("v=") }) + assertTrue(second.nextpage == null) + } + + @Test + fun `cursor is rejected after live visibility changes`() = withApp { + subscriptionsService.add(TEST_USER_ID, subscription(1)) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feedService.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(readPage(requestFeed(limit = 1)).nextpage) + + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.BadRequest, requestFeed(limit = 1, cursor = cursor).status) + } + + @Test + fun `hidden live streams do not remove finished recordings`() = withApp { + val channelService = mockk() + coEvery { channelService.getChannel(any(), null) } returns channel( + video(4_000L, url = "https://youtube.com/watch?v=live", live = true), + video(3_000L, url = "https://youtube.com/watch?v=replay").copy( + streamType = "post_live_stream", + isPostLive = true, + isLiveContent = true, + ), + ) + feedService = SubscriptionFeedService(subscriptionsService, channelService, FakeCacheService()) + subscriptionsService.add(TEST_USER_ID, subscription(1)) + settingsService.upsert(TEST_USER_ID, SettingsItem(hideSubscriptionLiveStreams = true)) + + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 30).status) + feedService.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("replay"), readPage(requestFeed(limit = 30)).videos.map { it.url.substringAfter("v=") }) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionFeedRoutes(feedService, auth, settingsService) } + } + block() + } + + private suspend fun ApplicationTestBuilder.requestFeed( + limit: Int, + cursor: String? = null, + ): HttpResponse = client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + } + + private suspend fun readPage(response: HttpResponse): SubscriptionFeedResponse { + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } +} From 815aec522fcdfd236e36a386b23dd26011f66de4 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 10 Aug 2026 10:36:30 +0200 Subject: [PATCH 21/22] fix: drop stale live videos from feeds --- .../services/HomeRecommendationPoolBuilder.kt | 4 +++- .../services/HomeRecommendationPoolCache.kt | 2 +- .../server/services/SubscriptionFeedBuilder.kt | 11 +++++++++-- .../server/HomeRecommendationPoolBuilderTest.kt | 16 ++++++++++++++++ .../server/SubscriptionFeedRoutesTest.kt | 16 ++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt index 2fb091b7..489a596d 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolBuilder.kt @@ -56,7 +56,9 @@ class HomeRecommendationPoolBuilder { if (video.url in profile.feedbackBlockedVideos || video.url in profile.implicitBlockedVideos) return@forEach if (video.uploaderUrl.isNotBlank() && video.uploaderUrl in profile.blockedChannels) return@forEach if (video.uploaderUrl.isNotBlank() && video.uploaderUrl in profile.feedbackBlockedChannels) return@forEach - if (!allowLive && HomeRecommendationLiveTitleDetector.isLiveLike(video.title)) return@forEach + if (!allowLive && (video.isLive || HomeRecommendationLiveTitleDetector.isLiveLike(video.title))) { + return@forEach + } val score = scorer(video, profile) val scored = HomeRecommendationScoredVideo(video = video, score = score, source = tagged.source) val current = byUrl[video.url] diff --git a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt index 229de660..17525b84 100644 --- a/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt +++ b/src/main/kotlin/dev/typetype/server/services/HomeRecommendationPoolCache.kt @@ -41,6 +41,6 @@ class HomeRecommendationPoolCache(private val cache: dev.typetype.server.cache.C companion object { private const val CACHE_TTL_SECONDS = 3_600L private const val STALE_TTL_SECONDS = 86_400L - private const val CACHE_VERSION = 8 + private const val CACHE_VERSION = 9 } } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index 16d87f35..bae327dc 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -37,9 +37,16 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private suspend fun fetchSubscription(channelUrl: String): SubscriptionSourceResult = coroutineScope { val channel = async { fetchVideos(channelUrl) } val live = if (isYoutubeUrl(channelUrl)) async { fetchVideos(channelUrl.toLivestreamsTabUrl()) } else null - val results = listOfNotNull(channel.await(), live?.await()) + val channelResult = channel.await() + val liveResult = live?.await() + val videos = if (liveResult == null) { + channelResult.videos + } else { + channelResult.videos.filterNot(VideoItem::isLive) + liveResult.videos + } + val results = listOfNotNull(channelResult, liveResult) SubscriptionSourceResult( - videos = mergeVideos(results.flatMap { it.videos }), + videos = mergeVideos(videos), successfulSources = results.count { it.success }, failedSources = results.count { !it.success }, ) diff --git a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt index d5da8d72..eea6d9f3 100644 --- a/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt +++ b/src/test/kotlin/dev/typetype/server/HomeRecommendationPoolBuilderTest.kt @@ -51,6 +51,22 @@ class HomeRecommendationPoolBuilderTest { assertTrue(pool.discovery.first().url.endsWith("/normal1")) } + @Test + fun `pool builder drops candidates marked live without relying on their title`() { + val profile = profile() + val discovery = listOf( + tagged( + video("live", "a", title = "Weekly tech roundup").copy(isLive = true), + HomeRecommendationSourceTag.DISCOVERY_THEME, + ), + tagged(video("normal", "b", title = "Weekly tech roundup"), HomeRecommendationSourceTag.DISCOVERY_THEME), + ) + + val pool = HomeRecommendationPoolBuilder().build(profile, emptyList(), discovery, context) + + assertEquals(listOf("https://yt.com/v/normal"), pool.discovery.map { it.url }) + } + @Test fun `pool builder excludes titles containing a blocked keyword`() { val profile = profile(blockedKeywords = setOf("SPONSORED")) diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt index 1bcddd7a..426e74ff 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionFeedRoutesTest.kt @@ -121,6 +121,22 @@ class SubscriptionFeedRoutesTest { assertNotNull(feed.generatedAt) } + @Test + fun `stale live from channel page is absent when streams tab no longer lists it`() = withApp { + val channelUrl = "https://www.youtube.com/channel/UC1" + val staleLiveUrl = "https://www.youtube.com/watch?v=private" + subscriptionsService.add(TEST_USER_ID, subscription(channelUrl, "Live channel")) + coEvery { channelService.getChannel(channelUrl, null) } returns channel( + video(-1L, url = staleLiveUrl, live = true), + video(3000L, url = "https://www.youtube.com/watch?v=normal"), + ) + coEvery { channelService.getChannel("$channelUrl/streams", null) } returns channel() + + val feed = buildAndRead() + + assertEquals(listOf("https://www.youtube.com/watch?v=normal"), feed.videos.map { it.url }) + } + @Test fun `cursor keeps pagination on one generation after refresh`() = withApp { subscriptionsService.add(TEST_USER_ID, subscription(1)) From 0d26e0960b7717e3c4f1108a507281a8da7702a5 Mon Sep 17 00:00:00 2001 From: Priveetee Date: Mon, 10 Aug 2026 18:39:30 +0200 Subject: [PATCH 22/22] chore: prepare server 1.5.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 99db84dc..3f348d8d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC kotlin.code.style=official -appVersion=1.4.0 +appVersion=1.5.0 systemProp.sun.net.client.defaultReadTimeout=180000 systemProp.sun.net.client.defaultConnectTimeout=60000