From 827ba345324c37513a95fbf347ca277bb8d3241b Mon Sep 17 00:00:00 2001 From: User Date: Thu, 6 Aug 2026 21:47:00 -0700 Subject: [PATCH 1/2] feat: make large subscription libraries manageable Introduce account-scoped named groups without changing the existing flat subscription record or global feed contract. Filtered feeds project from one shared snapshot and bind cursors to the selected group view, keeping refresh and cache behavior focused while preserving ungrouped subscriptions. Constraint: Start from dev and keep the first contribution Server-only Constraint: Preserve the existing global feed and ungrouped subscriptions Rejected: Per-group feed snapshots | adds cache invalidation and refresh fanout Confidence: high Scope-risk: moderate Directive: Keep filtered cursors bound to their subscription selection Tested: ./gradlew check shadowJar (997 tests, OpenAPI, coverage, fat JAR) Not-tested: Live frontend integration, deferred to the follow-up Frontend PR --- openapi.yaml | 8 + openapi/components/subscriptions.yaml | 27 +++ openapi/paths/subscriptions.yaml | 157 ++++++++++++++- .../dev/typetype/server/ServiceRegistry.kt | 2 + .../dev/typetype/server/db/DatabaseFactory.kt | 4 + .../SubscriptionGroupMembershipsTable.kt | 17 ++ .../db/tables/SubscriptionGroupsTable.kt | 18 ++ .../server/models/SubscriptionGroupItem.kt | 12 ++ .../SubscriptionGroupMembershipRequest.kt | 6 + .../server/models/SubscriptionGroupRequest.kt | 6 + .../server/routes/SubscriptionFeedRoutes.kt | 21 +- .../server/routes/SubscriptionGroupsRoutes.kt | 114 +++++++++++ .../routes/SubscriptionSelectionParameter.kt | 29 +++ .../server/routes/SubscriptionsRoutes.kt | 24 ++- .../typetype/server/routes/UserDataRoutes.kt | 10 +- .../PipePipeBackupPersisterService.kt | 2 + .../services/SubscriptionFeedBuilder.kt | 24 ++- .../services/SubscriptionFeedService.kt | 12 +- .../services/SubscriptionFeedSnapshot.kt | 39 +++- .../services/SubscriptionGroupResults.kt | 17 ++ .../services/SubscriptionGroupsService.kt | 182 ++++++++++++++++++ .../server/services/SubscriptionSelection.kt | 17 ++ .../server/services/SubscriptionsService.kt | 36 +++- .../services/TypeTypeBackupCoreRestore.kt | 2 + .../server/SubscriptionGroupFeedRoutesTest.kt | 138 +++++++++++++ .../server/SubscriptionGroupsRoutesTest.kt | 161 ++++++++++++++++ .../server/SubscriptionGroupsServiceTest.kt | 105 ++++++++++ .../dev/typetype/server/TestDatabase.kt | 4 + 28 files changed, 1174 insertions(+), 20 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt create mode 100644 src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt create mode 100644 src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt diff --git a/openapi.yaml b/openapi.yaml index 00121b5c..4d43308e 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -42,6 +42,10 @@ paths: /playlist: { $ref: ./openapi/paths/playlists.yaml#/Playlist } /saved-playlists: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylists } /saved-playlists/{id}: { $ref: ./openapi/paths/saved-playlists.yaml#/SavedPlaylist } + /subscriptions: { $ref: ./openapi/paths/subscriptions.yaml#/Subscriptions } + /subscriptions/groups: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroups } + /subscriptions/groups/{groupId}: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroup } + /subscriptions/groups/{groupId}/channels: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionGroupChannels } /subscriptions/feed: { $ref: ./openapi/paths/subscriptions.yaml#/SubscriptionFeed } /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings } /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup } @@ -136,6 +140,10 @@ components: $ref: ./openapi/components/media.yaml#/PublicPlaylistItem SavedPlaylistItem: { $ref: ./openapi/components/media.yaml#/SavedPlaylistItem } SavedPlaylistRequest: { $ref: ./openapi/components/media.yaml#/SavedPlaylistRequest } + SubscriptionItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionItem } + SubscriptionGroupItem: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupItem } + SubscriptionGroupRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupRequest } + SubscriptionGroupMembershipRequest: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } SubscriptionFeedResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedResponse } SubscriptionFeedPreparingResponse: { $ref: ./openapi/components/subscriptions.yaml#/SubscriptionFeedPreparingResponse } SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem } diff --git a/openapi/components/subscriptions.yaml b/openapi/components/subscriptions.yaml index b0e7ad59..cbca8e91 100644 --- a/openapi/components/subscriptions.yaml +++ b/openapi/components/subscriptions.yaml @@ -1,3 +1,30 @@ +SubscriptionItem: + type: object + required: [channelUrl, name, avatarUrl, subscribedAt] + properties: + channelUrl: { type: string, minLength: 1 } + name: { type: string } + avatarUrl: { type: string } + subscribedAt: { type: integer, format: int64 } +SubscriptionGroupItem: + type: object + required: [id, name, channelCount, createdAt, updatedAt] + properties: + id: { type: string, format: uuid } + name: { type: string, minLength: 1, maxLength: 100 } + channelCount: { type: integer, minimum: 0 } + createdAt: { type: integer, format: int64 } + updatedAt: { type: integer, format: int64 } +SubscriptionGroupRequest: + type: object + required: [name] + properties: + name: { type: string, minLength: 1, maxLength: 100 } +SubscriptionGroupMembershipRequest: + type: object + required: [channelUrl] + properties: + channelUrl: { type: string, minLength: 1 } SubscriptionFeedResponse: type: object required: [videos, nextpage, generation, generatedAt, refreshing] diff --git a/openapi/paths/subscriptions.yaml b/openapi/paths/subscriptions.yaml index f0dcc65c..08047e5e 100644 --- a/openapi/paths/subscriptions.yaml +++ b/openapi/paths/subscriptions.yaml @@ -1,8 +1,161 @@ +Subscriptions: + get: + tags: [user-data] + summary: List the current user's subscriptions + description: Omit both filters for the global list. Use groupId for one named group or ungrouped=true for subscriptions in no groups. + parameters: + - name: groupId + in: query + required: false + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + schema: { type: boolean, default: false } + responses: + '200': + description: The selected subscription projection. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Subscribe to a channel + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + responses: + '201': + description: Subscription created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Unsubscribe from a channel + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Subscription deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroups: + get: + tags: [user-data] + summary: List the current user's subscription groups + responses: + '200': + description: Account-scoped named groups. + content: + application/json: + schema: + type: array + items: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '401': { $ref: ../components/common.yaml#/JsonError } + post: + tags: [user-data] + summary: Create a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '201': + description: Group created. + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupItem } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroup: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Rename a subscription group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupRequest } + responses: + '204': { description: Group renamed. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + '409': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Delete a subscription group + responses: + '204': { description: Group and its memberships deleted. } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } +SubscriptionGroupChannels: + parameters: + - name: groupId + in: path + required: true + schema: { type: string, format: uuid } + put: + tags: [user-data] + summary: Add a subscribed channel to a group + requestBody: + required: true + content: + application/json: + schema: { $ref: ../components/subscriptions.yaml#/SubscriptionGroupMembershipRequest } + responses: + '204': { description: Membership exists. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } + delete: + tags: [user-data] + summary: Remove a subscribed channel from a group + parameters: + - name: url + in: query + required: true + schema: { type: string, minLength: 1 } + responses: + '204': { description: Membership deleted. } + '400': { $ref: ../components/common.yaml#/JsonError } + '401': { $ref: ../components/common.yaml#/JsonError } + '404': { $ref: ../components/common.yaml#/JsonError } SubscriptionFeed: get: tags: [user-data] summary: Read a stable page from the current user's subscription feed snapshot parameters: + - name: groupId + in: query + required: false + description: Restrict the snapshot projection to subscriptions in one account-owned group. + schema: { type: string, format: uuid } + - name: ungrouped + in: query + required: false + description: Restrict the snapshot projection to subscriptions in no groups. + schema: { type: boolean, default: false } - name: page in: query required: false @@ -16,7 +169,7 @@ SubscriptionFeed: - name: cursor in: query required: false - description: Opaque continuation returned in nextpage. + description: Opaque continuation returned in nextpage and bound to the selected subscription filter. schema: { type: string } responses: '200': @@ -45,6 +198,8 @@ SubscriptionFeed: $ref: ../components/common.yaml#/JsonError '401': $ref: ../components/common.yaml#/JsonError + '404': + $ref: ../components/common.yaml#/JsonError '409': description: The cursor references a snapshot generation that is no longer retained. headers: diff --git a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt index b0cff887..d071962d 100644 --- a/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt +++ b/src/main/kotlin/dev/typetype/server/ServiceRegistry.kt @@ -27,6 +27,7 @@ import dev.typetype.server.services.SubscriptionFeedService import dev.typetype.server.services.SubscriptionShortsBlendService import dev.typetype.server.services.SubscriptionShortsFeedService import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService import dev.typetype.server.services.SubscriptionFeedCacheInvalidation import dev.typetype.server.services.SubscriptionFeedCacheInvalidator import dev.typetype.server.services.TypeTypeBackupService @@ -80,6 +81,7 @@ internal class ServiceRegistry( val sabrSessionStore = extraction.sabrSessionStore val historyService = HistoryService() val subscriptionsService = SubscriptionsService() + val subscriptionGroupsService = SubscriptionGroupsService() val subscriptionFeedService = SubscriptionFeedService(subscriptionsService, channelService, cache) val subscriptionShortsFeedService = SubscriptionShortsFeedService( subscriptionsService, diff --git a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt index e99acb31..d4e4d323 100644 --- a/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt +++ b/src/main/kotlin/dev/typetype/server/db/DatabaseFactory.kt @@ -16,6 +16,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.UsersTable import dev.typetype.server.db.tables.UserAvatarsTable import dev.typetype.server.db.tables.WatchLaterTable @@ -53,6 +55,8 @@ object DatabaseFactory { AdminSettingsTable, HistoryTable, SubscriptionsTable, + SubscriptionGroupsTable, + SubscriptionGroupMembershipsTable, PlaylistsTable, PlaylistVideosTable, WatchLaterTable, diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt new file mode 100644 index 00000000..93931a89 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupMembershipsTable.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.ReferenceOption +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupMembershipsTable : Table("subscription_group_memberships") { + val groupId = text("group_id").references(SubscriptionGroupsTable.id, onDelete = ReferenceOption.CASCADE) + val userId = text("user_id") + val channelUrl = text("channel_url") + val addedAt = long("added_at") + + init { + index(false, userId, channelUrl) + } + + override val primaryKey = PrimaryKey(groupId, channelUrl) +} diff --git a/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt new file mode 100644 index 00000000..13b89b20 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/db/tables/SubscriptionGroupsTable.kt @@ -0,0 +1,18 @@ +package dev.typetype.server.db.tables + +import org.jetbrains.exposed.v1.core.Table + +object SubscriptionGroupsTable : Table("subscription_groups") { + val id = text("id") + val userId = text("user_id") + val name = text("name") + val normalizedName = text("normalized_name") + val createdAt = long("created_at") + val updatedAt = long("updated_at") + + init { + uniqueIndex(userId, normalizedName) + } + + override val primaryKey = PrimaryKey(id) +} diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt new file mode 100644 index 00000000..684382bb --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupItem.kt @@ -0,0 +1,12 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupItem( + val id: String, + val name: String, + val channelCount: Int, + val createdAt: Long, + val updatedAt: Long, +) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt new file mode 100644 index 00000000..9a2fcb5f --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupMembershipRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupMembershipRequest(val channelUrl: String) diff --git a/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt new file mode 100644 index 00000000..136b831c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/models/SubscriptionGroupRequest.kt @@ -0,0 +1,6 @@ +package dev.typetype.server.models + +import kotlinx.serialization.Serializable + +@Serializable +data class SubscriptionGroupRequest(val name: String) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt index 110d257c..53c81bb4 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionFeedRoutes.kt @@ -5,6 +5,8 @@ import dev.typetype.server.models.SubscriptionFeedPreparingResponse import dev.typetype.server.services.AuthService import dev.typetype.server.services.SubscriptionFeedPageResult import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.server.response.respond @@ -13,14 +15,29 @@ import io.ktor.server.routing.get private const val MAX_FEED_PAGE = 10_000 -fun Route.subscriptionFeedRoutes(feedService: SubscriptionFeedService, authService: AuthService) { +fun Route.subscriptionFeedRoutes( + feedService: SubscriptionFeedService, + authService: AuthService, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), +) { get("/subscriptions/feed") { call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + if (selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } val page = call.request.queryParameters["page"]?.toIntOrNull()?.coerceIn(0, MAX_FEED_PAGE) ?: 0 val limit = call.request.queryParameters["limit"]?.toIntOrNull()?.coerceIn(1, 100) ?: 30 val cursor = call.request.queryParameters["cursor"] call.response.headers.append(HttpHeaders.CacheControl, "no-store") - when (val result = feedService.getPage(userId, page, limit, cursor)) { + when (val result = feedService.getPage(userId, page, limit, cursor, selection = selection)) { is SubscriptionFeedPageResult.Ready -> call.respond(result.response) is SubscriptionFeedPageResult.Preparing -> { call.response.headers.append(HttpHeaders.RetryAfter, "1") diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt new file mode 100644 index 00000000..c80acb3d --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionGroupsRoutes.kt @@ -0,0 +1,114 @@ +package dev.typetype.server.routes + +import dev.typetype.server.models.ErrorResponse +import dev.typetype.server.models.SubscriptionGroupMembershipRequest +import dev.typetype.server.models.SubscriptionGroupRequest +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall +import io.ktor.server.request.receive +import io.ktor.server.response.respond +import io.ktor.server.routing.Route +import io.ktor.server.routing.delete +import io.ktor.server.routing.get +import io.ktor.server.routing.post +import io.ktor.server.routing.put + +fun Route.subscriptionGroupsRoutes(groupsService: SubscriptionGroupsService, authService: AuthService) { + get("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> call.respond(groupsService.getAll(userId)) } + } + post("/subscriptions/groups") { + call.withJwtAuth(authService) { userId -> + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.create(userId, request.name), created = true) + } + } + put("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = call.receiveGroupRequest() ?: return@withJwtAuth + call.respondGroupWrite(groupsService.rename(userId, groupId, request.name), created = false) + } + } + delete("/subscriptions/groups/{groupId}") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + if (groupsService.delete(userId, groupId)) call.respond(HttpStatusCode.NoContent) else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Subscription group not found", "subscription_group_not_found")) + } + } + } + put("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val request = runCatching { call.receive() }.getOrElse { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + } + if (request.channelUrl.isBlank()) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("channelUrl must not be blank")) + } + call.respondMembership(groupsService.addSubscription(userId, groupId, request.channelUrl)) + } + } + delete("/subscriptions/groups/{groupId}/channels") { + call.withJwtAuth(authService) { userId -> + val groupId = call.groupId() ?: return@withJwtAuth call.respondMissingGroupId() + val channelUrl = call.request.queryParameters["url"]?.takeIf(String::isNotBlank) + ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing channelUrl")) + call.respondMembership(groupsService.removeSubscription(userId, groupId, channelUrl)) + } + } +} + +private fun ApplicationCall.groupId(): String? = parameters["groupId"]?.takeIf(String::isNotBlank) + +private suspend fun ApplicationCall.receiveGroupRequest(): SubscriptionGroupRequest? = + runCatching { receive() }.getOrElse { + respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body")) + null + } + +private suspend fun ApplicationCall.respondGroupWrite(result: SubscriptionGroupWriteResult, created: Boolean) { + when (result) { + is SubscriptionGroupWriteResult.Success -> if (created) respond(HttpStatusCode.Created, result.group) else { + respond(HttpStatusCode.NoContent) + } + SubscriptionGroupWriteResult.InvalidName -> respond( + HttpStatusCode.BadRequest, + ErrorResponse("Group name must contain 1 to 100 characters", "subscription_group_invalid_name"), + ) + SubscriptionGroupWriteResult.DuplicateName -> respond( + HttpStatusCode.Conflict, + ErrorResponse("A subscription group with this name already exists", "subscription_group_name_conflict"), + ) + SubscriptionGroupWriteResult.NotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMembership(result: SubscriptionGroupMembershipResult) { + when (result) { + SubscriptionGroupMembershipResult.Success -> respond(HttpStatusCode.NoContent) + SubscriptionGroupMembershipResult.GroupNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + SubscriptionGroupMembershipResult.SubscriptionNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription not found", "subscription_not_found"), + ) + SubscriptionGroupMembershipResult.MembershipNotFound -> respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group membership not found", "subscription_group_membership_not_found"), + ) + } +} + +private suspend fun ApplicationCall.respondMissingGroupId() = + respond(HttpStatusCode.BadRequest, ErrorResponse("Missing groupId")) diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt new file mode 100644 index 00000000..a847aab0 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionSelectionParameter.kt @@ -0,0 +1,29 @@ +package dev.typetype.server.routes + +import dev.typetype.server.services.SubscriptionSelection +import io.ktor.server.application.ApplicationCall + +internal sealed interface SubscriptionSelectionParseResult { + data class Valid(val selection: SubscriptionSelection) : SubscriptionSelectionParseResult + data object Invalid : SubscriptionSelectionParseResult +} + +internal fun ApplicationCall.parseSubscriptionSelection(): SubscriptionSelectionParseResult { + val rawGroupId = request.queryParameters["groupId"] + val groupId = rawGroupId?.takeIf(String::isNotBlank) + if (rawGroupId != null && groupId == null) return SubscriptionSelectionParseResult.Invalid + val rawUngrouped = request.queryParameters["ungrouped"] + val ungrouped = when (rawUngrouped) { + null -> false + "true" -> true + "false" -> false + else -> return SubscriptionSelectionParseResult.Invalid + } + if (groupId != null && ungrouped) return SubscriptionSelectionParseResult.Invalid + val selection = when { + groupId != null -> SubscriptionSelection.Group(groupId) + ungrouped -> SubscriptionSelection.Ungrouped + else -> SubscriptionSelection.All + } + return SubscriptionSelectionParseResult.Valid(selection) +} diff --git a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt index 3d7b40b2..c8877500 100644 --- a/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/SubscriptionsRoutes.kt @@ -6,6 +6,8 @@ import dev.typetype.server.services.AuthService import dev.typetype.server.services.HomeRecommendationWarmup import dev.typetype.server.services.NoopHomeRecommendationWarmup import dev.typetype.server.services.SubscriptionsService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection import io.ktor.http.HttpStatusCode import io.ktor.server.application.ApplicationCall import io.ktor.server.request.receive @@ -18,9 +20,27 @@ import io.ktor.server.routing.post import java.net.URLDecoder import java.nio.charset.StandardCharsets -fun Route.subscriptionsRoutes(subscriptionsService: SubscriptionsService, authService: AuthService, warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup) { +fun Route.subscriptionsRoutes( + subscriptionsService: SubscriptionsService, + authService: AuthService, + warmupService: HomeRecommendationWarmup = NoopHomeRecommendationWarmup, + groupsService: SubscriptionGroupsService = SubscriptionGroupsService(), +) { get("/subscriptions") { - call.withJwtAuth(authService) { userId -> call.respond(subscriptionsService.getAll(userId)) } + call.withJwtAuth(authService) { userId -> + val parsed = call.parseSubscriptionSelection() + if (parsed !is SubscriptionSelectionParseResult.Valid) { + return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid subscription filter")) + } + val selection = parsed.selection + if (selection is SubscriptionSelection.Group && !groupsService.exists(userId, selection.id)) { + return@withJwtAuth call.respond( + HttpStatusCode.NotFound, + ErrorResponse("Subscription group not found", "subscription_group_not_found"), + ) + } + call.respond(subscriptionsService.getAll(userId, selection)) + } } post("/subscriptions") { call.withJwtAuth(authService) { userId -> diff --git a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt index eab6f335..5380e9eb 100644 --- a/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt +++ b/src/main/kotlin/dev/typetype/server/routes/UserDataRoutes.kt @@ -17,8 +17,14 @@ internal fun Route.userDataRoutes( restoreService: PipePipeBackupImporterService, ) { historyRoutes(svc.historyService, authService, svc.settingsService) - subscriptionsRoutes(svc.subscriptionsService, authService, svc.homeRecommendationWarmupService) - subscriptionFeedRoutes(svc.subscriptionFeedService, authService) + subscriptionGroupsRoutes(svc.subscriptionGroupsService, authService) + subscriptionsRoutes( + svc.subscriptionsService, + authService, + svc.homeRecommendationWarmupService, + svc.subscriptionGroupsService, + ) + subscriptionFeedRoutes(svc.subscriptionFeedService, authService, svc.subscriptionGroupsService) subscriptionShortsFeedRoutes(svc.subscriptionShortsFeedService, authService) playlistRoutes(svc.playlistService, authService, svc.videoMetadataRepairService) savedPlaylistRoutes(svc.savedPlaylistService, svc.publicPlaylistService, authService) diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt index 9378fb54..30020624 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt @@ -7,6 +7,7 @@ import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.ProgressTable import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.models.RestorePipePipeResultItem import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.deleteWhere @@ -38,6 +39,7 @@ class PipePipeBackupPersisterService { private fun clearUserData(userId: String) { HistoryTable.deleteWhere { HistoryTable.userId eq userId } + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt index 16d87f35..cfcee4d2 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedBuilder.kt @@ -22,13 +22,28 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic } catch (error: CancellationException) { throw error } catch (_: Throwable) { - SubscriptionSourceResult(emptyList(), successfulSources = 0, failedSources = 1) + SubscriptionSourceResult( + channelUrl = subscription.channelUrl, + videos = emptyList(), + successfulSources = 0, + failedSources = 1, + ) } } }.map { it.await() } - val videos = outcomes.flatMap { it.videos }.deduplicated() + val videosByKey = linkedMapOf() + val sourceChannelUrls = linkedMapOf>() + outcomes.forEach { outcome -> + outcome.videos.forEach { video -> + val key = video.subscriptionFeedKey() + val current = videosByKey[key] + if (current == null || video.isLive && !current.isLive) videosByKey[key] = video + sourceChannelUrls.getOrPut(key, ::linkedSetOf).add(outcome.channelUrl) + } + } SubscriptionFeedBuildResult( - videos = videos, + videos = videosByKey.values.toList(), + sourceChannelUrls = sourceChannelUrls.mapValues { it.value.toList() }, successfulSources = outcomes.sumOf { it.successfulSources }, failedSources = outcomes.sumOf { it.failedSources }, ) @@ -39,6 +54,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic val live = if (isYoutubeUrl(channelUrl)) async { fetchVideos(channelUrl.toLivestreamsTabUrl()) } else null val results = listOfNotNull(channel.await(), live?.await()) SubscriptionSourceResult( + channelUrl = channelUrl, videos = mergeVideos(results.flatMap { it.videos }), successfulSources = results.count { it.success }, failedSources = results.count { !it.success }, @@ -84,6 +100,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic private data class SourceFetchResult(val videos: List, val success: Boolean) private data class SubscriptionSourceResult( + val channelUrl: String, val videos: List, val successfulSources: Int, val failedSources: Int, @@ -98,6 +115,7 @@ internal class SubscriptionFeedBuilder(private val channelService: ChannelServic internal data class SubscriptionFeedBuildResult( val videos: List, + val sourceChannelUrls: Map>, val successfulSources: Int, val failedSources: Int, ) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt index 48a09fc2..01a9981f 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedService.kt @@ -36,6 +36,7 @@ class SubscriptionFeedService( page: Int, limit: Int, cursor: String?, + selection: SubscriptionSelection = SubscriptionSelection.All, requestId: String? = currentRequestId(), ): SubscriptionFeedPageResult { val current = store.current(userId) @@ -49,6 +50,9 @@ class SubscriptionFeedService( val cursorState = cursor?.let(SubscriptionFeedCursorCodec::decode) if (cursor != null && cursorState == null) return SubscriptionFeedPageResult.InvalidCursor if (cursorState != null && cursorState.limit != limit) return SubscriptionFeedPageResult.InvalidCursor + if (cursorState != null && cursorState.filterKey != selection.cursorKey) { + return SubscriptionFeedPageResult.InvalidCursor + } val snapshot = when { cursorState == null -> current cursorState.generation == current.generation -> current @@ -56,7 +60,12 @@ class SubscriptionFeedService( ?: return SubscriptionFeedPageResult.StaleGeneration } val offset = cursorState?.offset ?: page * limit - return SubscriptionFeedPageResult.Ready(snapshot.page(offset, limit, isRefreshing(userId))) + val selectedChannelUrls = if (selection == SubscriptionSelection.All) null else { + subscriptionsService.getChannelUrls(userId, selection) + } + return SubscriptionFeedPageResult.Ready( + snapshot.page(offset, limit, isRefreshing(userId), selection, selectedChannelUrls), + ) } suspend fun getFeed(userId: String, page: Int, limit: Int): SubscriptionFeedResponse = @@ -139,6 +148,7 @@ class SubscriptionFeedService( stale = false, videos = ordering.videos, livePromotedAt = ordering.livePromotedAt, + sourceChannelUrls = result.sourceChannelUrls, ) runCatching { store.publish(userId, snapshot) }.onFailure { logger.warn("subscription_feed event=publish_failed user={} error={}", userKey(userId), it.message) diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt index b48aa4a8..d1c2b935 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionFeedSnapshot.kt @@ -13,6 +13,7 @@ internal data class SubscriptionFeedSnapshot( val stale: Boolean, val videos: List, val livePromotedAt: Map = emptyMap(), + val sourceChannelUrls: Map> = emptyMap(), ) @Serializable @@ -20,13 +21,14 @@ private data class SubscriptionFeedCursor( val generation: Long, val offset: Int, val limit: Int, + val filterKey: String = SubscriptionSelection.All.cursorKey, ) internal object SubscriptionFeedCursorCodec { - fun encode(generation: Long, offset: Int, limit: Int): String { + fun encode(generation: Long, offset: Int, limit: Int, filterKey: String): String { val payload = CacheJson.encodeToString( SubscriptionFeedCursor.serializer(), - SubscriptionFeedCursor(generation, offset, limit), + SubscriptionFeedCursor(generation, offset, limit, filterKey), ) return Base64.getUrlEncoder().withoutPadding().encodeToString(payload.toByteArray()) } @@ -35,7 +37,7 @@ internal object SubscriptionFeedCursorCodec { val payload = String(Base64.getUrlDecoder().decode(value)) val cursor = CacheJson.decodeFromString(SubscriptionFeedCursor.serializer(), payload) cursor.takeIf { it.generation > 0L && it.offset >= 0 && it.limit in 1..100 } - ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit) } + ?.let { SubscriptionFeedCursorState(it.generation, it.offset, it.limit, it.filterKey) } }.getOrNull() } @@ -43,22 +45,26 @@ internal data class SubscriptionFeedCursorState( val generation: Long, val offset: Int, val limit: Int, + val filterKey: String, ) internal fun SubscriptionFeedSnapshot.page( offset: Int, limit: Int, refreshing: Boolean, + selection: SubscriptionSelection = SubscriptionSelection.All, + selectedChannelUrls: Set? = null, ): SubscriptionFeedResponse { - val from = offset.coerceAtMost(videos.size) - val to = minOf(from + limit, videos.size) - val nextpage = if (to < videos.size) { - SubscriptionFeedCursorCodec.encode(generation, to, limit) + val projectedVideos = projectedVideos(selection, selectedChannelUrls) + val from = offset.coerceAtMost(projectedVideos.size) + val to = minOf(from + limit, projectedVideos.size) + val nextpage = if (to < projectedVideos.size) { + SubscriptionFeedCursorCodec.encode(generation, to, limit, selection.cursorKey) } else { null } return SubscriptionFeedResponse( - videos = videos.subList(from, to), + videos = projectedVideos.subList(from, to), nextpage = nextpage, generation = generation, generatedAt = generatedAt, @@ -66,6 +72,23 @@ internal fun SubscriptionFeedSnapshot.page( ) } +private fun SubscriptionFeedSnapshot.projectedVideos( + selection: SubscriptionSelection, + selectedChannelUrls: Set?, +): List { + if (selection == SubscriptionSelection.All) return videos + val allowed = selectedChannelUrls.orEmpty() + if (allowed.isEmpty()) return emptyList() + return videos.filter { video -> + val sources = sourceChannelUrls[video.subscriptionFeedKey()] + if (sources != null) { + sources.any { ChannelUrlCanonicalizer.canonicalize(it) in allowed } + } else { + ChannelUrlCanonicalizer.canonicalize(video.uploaderUrl) in allowed + } + } +} + internal sealed interface SubscriptionFeedPageResult { data class Ready(val response: SubscriptionFeedResponse) : SubscriptionFeedPageResult data class Preparing(val retryAfterMs: Long) : SubscriptionFeedPageResult diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt new file mode 100644 index 00000000..6c19f560 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupResults.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +import dev.typetype.server.models.SubscriptionGroupItem + +sealed interface SubscriptionGroupWriteResult { + data class Success(val group: SubscriptionGroupItem) : SubscriptionGroupWriteResult + data object InvalidName : SubscriptionGroupWriteResult + data object DuplicateName : SubscriptionGroupWriteResult + data object NotFound : SubscriptionGroupWriteResult +} + +sealed interface SubscriptionGroupMembershipResult { + data object Success : SubscriptionGroupMembershipResult + data object GroupNotFound : SubscriptionGroupMembershipResult + data object SubscriptionNotFound : SubscriptionGroupMembershipResult + data object MembershipNotFound : SubscriptionGroupMembershipResult +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt new file mode 100644 index 00000000..23887b2c --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupsService.kt @@ -0,0 +1,182 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable +import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.models.SubscriptionGroupItem +import org.jetbrains.exposed.v1.core.ResultRow +import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insertIgnore +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.update +import java.sql.SQLException +import java.util.Locale +import java.util.UUID + +class SubscriptionGroupsService { + suspend fun getAll(userId: String): List = DatabaseFactory.query { + val counts = SubscriptionGroupMembershipsTable.selectAll() + .where { SubscriptionGroupMembershipsTable.userId eq userId } + .groupingBy { it[SubscriptionGroupMembershipsTable.groupId] } + .eachCount() + SubscriptionGroupsTable.selectAll() + .where { SubscriptionGroupsTable.userId eq userId } + .orderBy(SubscriptionGroupsTable.createdAt to SortOrder.DESC) + .map { it.toItem(counts[it[SubscriptionGroupsTable.id]] ?: 0) } + } + + suspend fun exists(userId: String, groupId: String): Boolean = DatabaseFactory.query { + groupExists(userId, groupId) + } + + suspend fun create(userId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return DatabaseFactory.query { + if (nameExists(userId, normalizedName)) return@query SubscriptionGroupWriteResult.DuplicateName + val id = UUID.randomUUID().toString() + val now = System.currentTimeMillis() + val inserted = SubscriptionGroupsTable.insertIgnore { + it[SubscriptionGroupsTable.id] = id + it[SubscriptionGroupsTable.userId] = userId + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[createdAt] = now + it[updatedAt] = now + }.insertedCount + if (inserted == 0) SubscriptionGroupWriteResult.DuplicateName else { + SubscriptionGroupWriteResult.Success(SubscriptionGroupItem(id, name, 0, now, now)) + } + } + } + + suspend fun rename(userId: String, groupId: String, rawName: String): SubscriptionGroupWriteResult { + val name = normalizeDisplayName(rawName) ?: return SubscriptionGroupWriteResult.InvalidName + val normalizedName = normalizeUniqueName(name) + return try { + DatabaseFactory.query { + val current = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.singleOrNull() ?: return@query SubscriptionGroupWriteResult.NotFound + val duplicate = SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any { it[SubscriptionGroupsTable.id] != groupId } + if (duplicate) return@query SubscriptionGroupWriteResult.DuplicateName + val now = System.currentTimeMillis() + SubscriptionGroupsTable.update({ + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }) { + it[SubscriptionGroupsTable.name] = name + it[SubscriptionGroupsTable.normalizedName] = normalizedName + it[updatedAt] = now + } + val count = membershipCount(userId, groupId) + SubscriptionGroupWriteResult.Success( + SubscriptionGroupItem(groupId, name, count, current[SubscriptionGroupsTable.createdAt], now), + ) + } + } catch (error: Throwable) { + if (error.isUniqueConstraintViolation()) SubscriptionGroupWriteResult.DuplicateName else throw error + } + } + + suspend fun delete(userId: String, groupId: String): Boolean = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query false + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + } + SubscriptionGroupsTable.deleteWhere { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + } > 0 + } + + suspend fun addSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val subscriptionExists = SubscriptionsTable.selectAll().where { + (SubscriptionsTable.userId eq userId) and (SubscriptionsTable.channelUrl eq channelUrl) + }.any() + if (!subscriptionExists) return@query SubscriptionGroupMembershipResult.SubscriptionNotFound + SubscriptionGroupMembershipsTable.insertIgnore { + it[SubscriptionGroupMembershipsTable.groupId] = groupId + it[SubscriptionGroupMembershipsTable.userId] = userId + it[SubscriptionGroupMembershipsTable.channelUrl] = channelUrl + it[addedAt] = System.currentTimeMillis() + } + SubscriptionGroupMembershipResult.Success + } + + suspend fun removeSubscription( + userId: String, + groupId: String, + rawChannelUrl: String, + ): SubscriptionGroupMembershipResult = DatabaseFactory.query { + if (!groupExists(userId, groupId)) return@query SubscriptionGroupMembershipResult.GroupNotFound + val channelUrl = ChannelUrlCanonicalizer.canonicalize(rawChannelUrl) + val deleted = SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq channelUrl) + } + if (deleted > 0) SubscriptionGroupMembershipResult.Success else { + SubscriptionGroupMembershipResult.MembershipNotFound + } + } + + suspend fun getChannelUrls(userId: String, groupId: String): List = DatabaseFactory.query { + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.groupId eq groupId) and + (SubscriptionGroupMembershipsTable.userId eq userId) + }.orderBy(SubscriptionGroupMembershipsTable.addedAt to SortOrder.DESC) + .map { it[SubscriptionGroupMembershipsTable.channelUrl] } + } + + private fun groupExists(userId: String, groupId: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.id eq groupId) and (SubscriptionGroupsTable.userId eq userId) + }.any() + + private fun nameExists(userId: String, normalizedName: String): Boolean = + SubscriptionGroupsTable.selectAll().where { + (SubscriptionGroupsTable.userId eq userId) and + (SubscriptionGroupsTable.normalizedName eq normalizedName) + }.any() + + private fun membershipCount(userId: String, groupId: String): Int = + SubscriptionGroupMembershipsTable.selectAll().where { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq groupId) + }.count().toInt() + + private fun ResultRow.toItem(channelCount: Int): SubscriptionGroupItem = SubscriptionGroupItem( + id = this[SubscriptionGroupsTable.id], + name = this[SubscriptionGroupsTable.name], + channelCount = channelCount, + createdAt = this[SubscriptionGroupsTable.createdAt], + updatedAt = this[SubscriptionGroupsTable.updatedAt], + ) + + private fun normalizeDisplayName(value: String): String? = + value.trim().takeIf { it.length in 1..MAX_GROUP_NAME_LENGTH } + + private fun normalizeUniqueName(value: String): String = value.lowercase(Locale.ROOT) + + private fun Throwable.isUniqueConstraintViolation(): Boolean = generateSequence(this) { it.cause } + .filterIsInstance() + .any { it.sqlState == UNIQUE_VIOLATION_SQL_STATE } + + companion object { + const val MAX_GROUP_NAME_LENGTH = 100 + private const val UNIQUE_VIOLATION_SQL_STATE = "23505" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt new file mode 100644 index 00000000..cbdf5d99 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionSelection.kt @@ -0,0 +1,17 @@ +package dev.typetype.server.services + +sealed interface SubscriptionSelection { + val cursorKey: String + + data object All : SubscriptionSelection { + override val cursorKey: String = "all" + } + + data object Ungrouped : SubscriptionSelection { + override val cursorKey: String = "ungrouped" + } + + data class Group(val id: String) : SubscriptionSelection { + override val cursorKey: String = "group:$id" + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt index 887a931e..21d022de 100644 --- a/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionsService.kt @@ -1,6 +1,7 @@ package dev.typetype.server.services import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.db.tables.SubscriptionsTable import dev.typetype.server.models.SubscriptionItem import org.jetbrains.exposed.v1.core.ResultRow @@ -13,14 +14,22 @@ import org.jetbrains.exposed.v1.jdbc.selectAll class SubscriptionsService { - suspend fun getAll(userId: String): List = DatabaseFactory.query { + suspend fun getAll( + userId: String, + selection: SubscriptionSelection = SubscriptionSelection.All, + ): List = DatabaseFactory.query { + val selectedUrls = selectedChannelUrls(userId, selection) val items = SubscriptionsTable.selectAll() .where { SubscriptionsTable.userId eq userId } .orderBy(SubscriptionsTable.subscribedAt to SortOrder.DESC) .map { it.toItem() } + .filter { selection == SubscriptionSelection.All || it.channelUrl in selectedUrls } SubscriptionAvatarRepairer.repair(userId = userId, items = items) } + suspend fun getChannelUrls(userId: String, selection: SubscriptionSelection): Set = + DatabaseFactory.query { selectedChannelUrls(userId, selection) } + suspend fun add(userId: String, item: SubscriptionItem): SubscriptionItem { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(item.channelUrl) val now = System.currentTimeMillis() @@ -38,9 +47,34 @@ class SubscriptionsService { suspend fun delete(userId: String, channelUrl: String): Boolean = DatabaseFactory.query { val canonicalUrl = ChannelUrlCanonicalizer.canonicalize(channelUrl) + SubscriptionGroupMembershipsTable.deleteWhere { + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.channelUrl eq canonicalUrl) + } SubscriptionsTable.deleteWhere { SubscriptionsTable.channelUrl eq canonicalUrl and (SubscriptionsTable.userId eq userId) } > 0 } + private fun selectedChannelUrls(userId: String, selection: SubscriptionSelection): Set { + val all = SubscriptionsTable.selectAll() + .where { SubscriptionsTable.userId eq userId } + .mapTo(linkedSetOf()) { ChannelUrlCanonicalizer.canonicalize(it[SubscriptionsTable.channelUrl]) } + if (selection == SubscriptionSelection.All) return all + val memberships = SubscriptionGroupMembershipsTable.selectAll().where { + when (selection) { + SubscriptionSelection.All -> SubscriptionGroupMembershipsTable.userId eq userId + SubscriptionSelection.Ungrouped -> SubscriptionGroupMembershipsTable.userId eq userId + is SubscriptionSelection.Group -> + (SubscriptionGroupMembershipsTable.userId eq userId) and + (SubscriptionGroupMembershipsTable.groupId eq selection.id) + } + }.mapTo(mutableSetOf()) { it[SubscriptionGroupMembershipsTable.channelUrl] } + return when (selection) { + SubscriptionSelection.All -> all + SubscriptionSelection.Ungrouped -> all - memberships + is SubscriptionSelection.Group -> all intersect memberships + } + } + private fun ResultRow.toItem() = SubscriptionItem( channelUrl = ChannelUrlCanonicalizer.canonicalize(this[SubscriptionsTable.channelUrl]), name = this[SubscriptionsTable.name], diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt index f8f5b715..f2f26acd 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt @@ -4,6 +4,7 @@ import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.models.HistoryItem import dev.typetype.server.models.PlaylistItem import dev.typetype.server.models.SubscriptionItem @@ -14,6 +15,7 @@ import java.util.UUID internal object TypeTypeBackupCoreRestore { fun subscriptions(userId: String, items: List): Int { + SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } SubscriptionsTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> this[SubscriptionsTable.userId] = userId diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt new file mode 100644 index 00000000..d9aed6fd --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupFeedRoutesTest.kt @@ -0,0 +1,138 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionFeedResponse +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionFeedRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionFeedService +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import io.mockk.coEvery +import io.mockk.mockk +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupFeedRoutesTest { + private val subscriptions = SubscriptionsService() + private val groups = SubscriptionGroupsService() + private lateinit var feed: SubscriptionFeedService + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() { + TestDatabase.truncateAll() + feed = SubscriptionFeedService(subscriptions, FakeChannelService(), FakeCacheService()) + } + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { subscriptionFeedRoutes(feed, auth, groups) } + } + block() + } + + @Test + fun `group and ungrouped feeds project one shared global snapshot`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + assertEquals( + SubscriptionGroupMembershipResult.Success, + groups.addSubscription(TEST_USER_ID, group.id, channel("one")), + ) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(listOf("${channel("one")}/video"), requestReadyFeed(groupId = group.id).videos.map { it.url }) + assertEquals(listOf("${channel("two")}/video"), requestReadyFeed(ungrouped = true).videos.map { it.url }) + assertEquals(2, requestReadyFeed().videos.size) + } + + @Test + fun `cursor cannot be reused with another subscription filter`() = withApp { + subscriptions.add(TEST_USER_ID, subscription("one")) + subscriptions.add(TEST_USER_ID, subscription("two")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, channel("one")) + assertEquals(HttpStatusCode.Accepted, requestFeed(limit = 1).status) + feed.awaitRefresh(TEST_USER_ID) + val cursor = requireNotNull(requestReadyFeed(limit = 1).nextpage) + + val response = requestFeed(limit = 1, cursor = cursor, groupId = group.id) + + assertEquals(HttpStatusCode.BadRequest, response.status) + assertTrue(response.bodyAsText().contains("subscription_feed_invalid_cursor")) + } + + @Test + fun `group feed follows the fetched subscription source when uploader url differs`() = withApp { + val sourceUrl = channel("one") + subscriptions.add(TEST_USER_ID, subscription("one")) + val group = (groups.create(TEST_USER_ID, "Work") as SubscriptionGroupWriteResult.Success).group + groups.addSubscription(TEST_USER_ID, group.id, sourceUrl) + val channelService = mockk() + coEvery { channelService.getChannel(sourceUrl, null) } returns SubscriptionFeedTestFixtures.channel( + SubscriptionFeedTestFixtures.video(1_000L, channel = "different-canonical-uploader"), + ) + feed = SubscriptionFeedService(subscriptions, channelService, FakeCacheService()) + + assertEquals(HttpStatusCode.Accepted, requestFeed(groupId = group.id).status) + feed.awaitRefresh(TEST_USER_ID) + + assertEquals(1, requestReadyFeed(groupId = group.id).videos.size) + } + + private suspend fun ApplicationTestBuilder.requestReadyFeed( + limit: Int = 30, + groupId: String? = null, + ungrouped: Boolean = false, + ): SubscriptionFeedResponse { + val response = requestFeed(limit = limit, groupId = groupId, ungrouped = ungrouped) + assertEquals(HttpStatusCode.OK, response.status) + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.requestFeed( + limit: Int = 30, + cursor: String? = null, + groupId: String? = null, + ungrouped: Boolean = false, + ): HttpResponse = client.get("/subscriptions/feed") { + header(HttpHeaders.Authorization, "Bearer test-jwt") + parameter("limit", limit) + cursor?.let { parameter("cursor", it) } + groupId?.let { parameter("groupId", it) } + if (ungrouped) parameter("ungrouped", true) + } + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://example.com/channel/$id" +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt new file mode 100644 index 00000000..70dd04b3 --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsRoutesTest.kt @@ -0,0 +1,161 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionGroupItem +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.routes.subscriptionGroupsRoutes +import dev.typetype.server.routes.subscriptionsRoutes +import dev.typetype.server.services.AuthService +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionsService +import io.ktor.client.request.delete +import io.ktor.client.request.get +import io.ktor.client.request.header +import io.ktor.client.request.parameter +import io.ktor.client.request.post +import io.ktor.client.request.put +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.HttpStatusCode +import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.install +import io.ktor.server.plugins.contentnegotiation.ContentNegotiation +import io.ktor.server.routing.routing +import io.ktor.server.testing.ApplicationTestBuilder +import io.ktor.server.testing.testApplication +import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupsRoutesTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + private val auth = AuthService.fixed(TEST_USER_ID) + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + private fun withApp(block: suspend ApplicationTestBuilder.() -> Unit) = testApplication { + application { + install(ContentNegotiation) { json() } + routing { + subscriptionGroupsRoutes(groups, auth) + subscriptionsRoutes(subscriptions, auth, groupsService = groups) + } + } + block() + } + + @Test + fun `group routes require authentication`() = withApp { + assertEquals(HttpStatusCode.Unauthorized, client.get("/subscriptions/groups").status) + } + + @Test + fun `groups can be created listed renamed and deleted`() = withApp { + val create = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + } + assertEquals(HttpStatusCode.Created, create.status) + val group = Json.decodeFromString(create.bodyAsText()) + + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Work\"")) + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}") { + authorizeJson() + setBody("""{"name":"Research"}""") + }.status) + assertTrue(authorizedGet("/subscriptions/groups").bodyAsText().contains("\"name\":\"Research\"")) + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}") { authorize() }.status) + assertEquals("[]", authorizedGet("/subscriptions/groups").bodyAsText()) + } + + @Test + fun `blank and duplicate group names are rejected`() = withApp { + assertEquals(HttpStatusCode.BadRequest, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":" "}""") + }.status) + assertEquals(HttpStatusCode.Created, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"Work"}""") + }.status) + assertEquals(HttpStatusCode.Conflict, client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"work"}""") + }.status) + } + + @Test + fun `membership drives grouped and ungrouped subscription projections`() = withApp { + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("one"), "One", "")) + subscriptions.add(TEST_USER_ID, SubscriptionItem(channel("two"), "Two", "")) + val group = createGroup("Work") + + assertEquals(HttpStatusCode.NoContent, client.put("/subscriptions/groups/${group.id}/channels") { + authorizeJson() + setBody("""{"channelUrl":"${channel("one")}"}""") + }.status) + + val grouped = authorizedGet("/subscriptions") { parameter("groupId", group.id) } + assertTrue(grouped.bodyAsText().contains(channel("one"))) + assertTrue(!grouped.bodyAsText().contains(channel("two"))) + val ungrouped = authorizedGet("/subscriptions") { parameter("ungrouped", true) } + assertTrue(!ungrouped.bodyAsText().contains(channel("one"))) + assertTrue(ungrouped.bodyAsText().contains(channel("two"))) + + assertEquals(HttpStatusCode.NoContent, client.delete("/subscriptions/groups/${group.id}/channels") { + authorize() + parameter("url", channel("one")) + }.status) + assertTrue(authorizedGet("/subscriptions") { parameter("ungrouped", true) }.bodyAsText().contains(channel("one"))) + } + + @Test + fun `invalid or inaccessible filters fail explicitly`() = withApp { + assertEquals(HttpStatusCode.BadRequest, authorizedGet("/subscriptions") { + parameter("groupId", "group") + parameter("ungrouped", true) + }.status) + assertEquals(HttpStatusCode.NotFound, authorizedGet("/subscriptions") { + parameter("groupId", "missing") + }.status) + } + + private suspend fun ApplicationTestBuilder.createGroup(name: String): SubscriptionGroupItem { + val response = client.post("/subscriptions/groups") { + authorizeJson() + setBody("""{"name":"$name"}""") + } + return Json.decodeFromString(response.bodyAsText()) + } + + private suspend fun ApplicationTestBuilder.authorizedGet( + path: String, + configure: io.ktor.client.request.HttpRequestBuilder.() -> Unit = {}, + ) = client.get(path) { + authorize() + configure() + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorize() { + header(HttpHeaders.Authorization, "Bearer test-jwt") + } + + private fun io.ktor.client.request.HttpRequestBuilder.authorizeJson() { + authorize() + header(HttpHeaders.ContentType, ContentType.Application.Json.toString()) + } + + private fun channel(id: String) = "https://yt.com/channel/$id" +} diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt new file mode 100644 index 00000000..0d32240d --- /dev/null +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -0,0 +1,105 @@ +package dev.typetype.server + +import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.services.SubscriptionGroupMembershipResult +import dev.typetype.server.services.SubscriptionGroupWriteResult +import dev.typetype.server.services.SubscriptionGroupsService +import dev.typetype.server.services.SubscriptionSelection +import dev.typetype.server.services.SubscriptionsService +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class SubscriptionGroupsServiceTest { + private val groups = SubscriptionGroupsService() + private val subscriptions = SubscriptionsService() + + companion object { + @BeforeAll + @JvmStatic + fun initDb() = TestDatabase.setup() + } + + @BeforeEach + fun clean() = TestDatabase.truncateAll() + + @Test + fun `group names are normalized unique and account scoped`() = runTest { + val group = groups.create("user-a", " Work ").createdGroup() + + assertEquals("Work", group.name) + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.create("user-a", "work")) + assertTrue(groups.create("user-b", "work") is SubscriptionGroupWriteResult.Success) + assertFalse(groups.exists("user-b", group.id)) + assertEquals( + SubscriptionGroupWriteResult.NotFound, + groups.rename("user-b", group.id, "Other"), + ) + groups.create("user-a", "Other") + assertEquals(SubscriptionGroupWriteResult.DuplicateName, groups.rename("user-a", group.id, "OTHER")) + } + + @Test + fun `a subscription can belong to multiple groups while ungrouped stays distinct`() = runTest { + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + subscriptions.add("user", subscription("three")) + val first = groups.create("user", "First").createdGroup() + val second = groups.create("user", "Second").createdGroup() + + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", first.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("one"))) + assertEquals(SubscriptionGroupMembershipResult.Success, groups.addSubscription("user", second.id, channel("two"))) + assertEquals(1, groups.getAll("user").first { it.id == first.id }.channelCount) + + assertEquals( + listOf(channel("one")), + subscriptions.getAll("user", SubscriptionSelection.Group(first.id)).map { it.channelUrl }, + ) + assertEquals( + setOf(channel("one"), channel("two")), + subscriptions.getAll("user", SubscriptionSelection.Group(second.id)).map { it.channelUrl }.toSet(), + ) + assertEquals( + listOf(channel("three")), + subscriptions.getAll("user", SubscriptionSelection.Ungrouped).map { it.channelUrl }, + ) + } + + @Test + fun `membership requires both the users group and subscription`() = runTest { + val group = groups.create("user-a", "A").createdGroup() + subscriptions.add("user-b", subscription("shared")) + + assertEquals( + SubscriptionGroupMembershipResult.SubscriptionNotFound, + groups.addSubscription("user-a", group.id, channel("shared")), + ) + assertEquals( + SubscriptionGroupMembershipResult.GroupNotFound, + groups.addSubscription("user-b", group.id, channel("shared")), + ) + } + + @Test + fun `deleting a subscription removes its memberships`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + groups.addSubscription("user", group.id, channel("one")) + + assertTrue(subscriptions.delete("user", channel("one"))) + + assertEquals(emptyList(), groups.getChannelUrls("user", group.id)) + } + + private fun SubscriptionGroupWriteResult.createdGroup() = + (this as SubscriptionGroupWriteResult.Success).group + + private fun subscription(id: String) = SubscriptionItem(channel(id), id, "") + + private fun channel(id: String) = "https://yt.com/channel/$id" +} diff --git a/src/test/kotlin/dev/typetype/server/TestDatabase.kt b/src/test/kotlin/dev/typetype/server/TestDatabase.kt index f55367ae..2a5410a3 100644 --- a/src/test/kotlin/dev/typetype/server/TestDatabase.kt +++ b/src/test/kotlin/dev/typetype/server/TestDatabase.kt @@ -20,6 +20,8 @@ import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SettingsTable import dev.typetype.server.db.tables.SessionsTable import dev.typetype.server.db.tables.SubscriptionsTable +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import dev.typetype.server.db.tables.SubscriptionGroupsTable import dev.typetype.server.db.tables.YoutubeTakeoutImportJobsTable import dev.typetype.server.db.tables.YoutubeTakeoutPlaylistKeysTable import dev.typetype.server.db.tables.YoutubeSessionPairingsTable @@ -94,6 +96,8 @@ object TestDatabase { HistoryTable.deleteAll() FavoritesTable.deleteAll() SettingsTable.deleteAll() + SubscriptionGroupMembershipsTable.deleteAll() + SubscriptionGroupsTable.deleteAll() SubscriptionsTable.deleteAll() WatchLaterTable.deleteAll() ProgressTable.deleteAll() From a2609067d2dab73554450da499649035d3756afb Mon Sep 17 00:00:00 2001 From: User Date: Thu, 6 Aug 2026 21:49:34 -0700 Subject: [PATCH 2/2] fix: preserve valid group assignments across subscription restores Prune only memberships whose channels disappear from a replacement import or restore. This avoids orphaned rows without discarding assignments for channels that remain subscribed. Constraint: Existing backup formats do not carry subscription-group metadata Rejected: Clear every membership during restore | loses still-valid assignments Confidence: high Scope-risk: narrow Directive: Canonicalize restored URLs before pruning group memberships Tested: ./gradlew check shadowJar (998 tests, OpenAPI, coverage, fat JAR) Not-tested: Restoring group definitions into a different account --- .../PipePipeBackupPersisterService.kt | 3 +-- .../SubscriptionGroupMembershipCleaner.kt | 19 +++++++++++++++++++ .../services/TypeTypeBackupCoreRestore.kt | 3 +-- .../server/SubscriptionGroupsServiceTest.kt | 15 +++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt diff --git a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt index 30020624..4e965014 100644 --- a/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt +++ b/src/main/kotlin/dev/typetype/server/services/PipePipeBackupPersisterService.kt @@ -7,7 +7,6 @@ import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.ProgressTable import dev.typetype.server.db.tables.SearchHistoryTable import dev.typetype.server.db.tables.SubscriptionsTable -import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.models.RestorePipePipeResultItem import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.jdbc.deleteWhere @@ -23,6 +22,7 @@ class PipePipeBackupPersisterService { .toMap() val history = insertHistory(userId, snapshot.history, avatarsByChannel) val subscriptions = insertSubscriptions(userId, snapshot.subscriptions) + SubscriptionGroupMembershipCleaner.retain(userId, snapshot.subscriptions.map { it.url }) val (playlists, playlistVideos) = insertPlaylists(userId, snapshot.playlists) val progress = insertProgress(userId, snapshot.progress) val searchHistory = insertSearchHistory(userId, snapshot.searchHistory) @@ -39,7 +39,6 @@ class PipePipeBackupPersisterService { private fun clearUserData(userId: String) { HistoryTable.deleteWhere { HistoryTable.userId eq userId } - SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } PlaylistVideosTable.deleteWhere { PlaylistVideosTable.userId eq userId } PlaylistsTable.deleteWhere { PlaylistsTable.userId eq userId } diff --git a/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt new file mode 100644 index 00000000..d78f7833 --- /dev/null +++ b/src/main/kotlin/dev/typetype/server/services/SubscriptionGroupMembershipCleaner.kt @@ -0,0 +1,19 @@ +package dev.typetype.server.services + +import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.notInList +import org.jetbrains.exposed.v1.jdbc.deleteWhere + +internal object SubscriptionGroupMembershipCleaner { + fun retain(userId: String, channelUrls: Collection) { + val retained = channelUrls.mapTo(linkedSetOf(), ChannelUrlCanonicalizer::canonicalize) + SubscriptionGroupMembershipsTable.deleteWhere { + val ownedByUser = SubscriptionGroupMembershipsTable.userId eq userId + if (retained.isEmpty()) ownedByUser else { + ownedByUser and (SubscriptionGroupMembershipsTable.channelUrl notInList retained) + } + } + } +} diff --git a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt index f2f26acd..7106218c 100644 --- a/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt +++ b/src/main/kotlin/dev/typetype/server/services/TypeTypeBackupCoreRestore.kt @@ -4,7 +4,6 @@ import dev.typetype.server.db.tables.HistoryTable import dev.typetype.server.db.tables.PlaylistVideosTable import dev.typetype.server.db.tables.PlaylistsTable import dev.typetype.server.db.tables.SubscriptionsTable -import dev.typetype.server.db.tables.SubscriptionGroupMembershipsTable import dev.typetype.server.models.HistoryItem import dev.typetype.server.models.PlaylistItem import dev.typetype.server.models.SubscriptionItem @@ -15,7 +14,6 @@ import java.util.UUID internal object TypeTypeBackupCoreRestore { fun subscriptions(userId: String, items: List): Int { - SubscriptionGroupMembershipsTable.deleteWhere { SubscriptionGroupMembershipsTable.userId eq userId } SubscriptionsTable.deleteWhere { SubscriptionsTable.userId eq userId } SubscriptionsTable.batchInsert(items, shouldReturnGeneratedValues = false) { item -> this[SubscriptionsTable.userId] = userId @@ -24,6 +22,7 @@ internal object TypeTypeBackupCoreRestore { this[SubscriptionsTable.avatarUrl] = item.avatarUrl this[SubscriptionsTable.subscribedAt] = item.subscribedAt } + SubscriptionGroupMembershipCleaner.retain(userId, items.map(SubscriptionItem::channelUrl)) return items.size } diff --git a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt index 0d32240d..d03bce44 100644 --- a/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt +++ b/src/test/kotlin/dev/typetype/server/SubscriptionGroupsServiceTest.kt @@ -1,6 +1,8 @@ package dev.typetype.server import dev.typetype.server.models.SubscriptionItem +import dev.typetype.server.db.DatabaseFactory +import dev.typetype.server.services.SubscriptionGroupMembershipCleaner import dev.typetype.server.services.SubscriptionGroupMembershipResult import dev.typetype.server.services.SubscriptionGroupWriteResult import dev.typetype.server.services.SubscriptionGroupsService @@ -96,6 +98,19 @@ class SubscriptionGroupsServiceTest { assertEquals(emptyList(), groups.getChannelUrls("user", group.id)) } + @Test + fun `replacement imports retain only memberships for subscriptions still present`() = runTest { + val group = groups.create("user", "Group").createdGroup() + subscriptions.add("user", subscription("one")) + subscriptions.add("user", subscription("two")) + groups.addSubscription("user", group.id, channel("one")) + groups.addSubscription("user", group.id, channel("two")) + + DatabaseFactory.query { SubscriptionGroupMembershipCleaner.retain("user", listOf(channel("one"))) } + + assertEquals(listOf(channel("one")), groups.getChannelUrls("user", group.id)) + } + private fun SubscriptionGroupWriteResult.createdGroup() = (this as SubscriptionGroupWriteResult.Success).group