diff --git a/docs/openapi.yaml b/docs/openapi.yaml index dea67fb78..e91c96023 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -86,6 +86,20 @@ $defs: minLength: 1 maxLength: 50 example: dororon-enma-kun + animecountdown: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 + animenewsnetwork: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 anisearch: oneOf: - type: "null" @@ -115,6 +129,27 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 + media: + oneOf: + - type: "null" + - type: string + minLength: 0 + maxLength: 10 + example: TV + myanimelist: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 + simkl: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1337 themoviedb: oneOf: - type: "null" @@ -122,6 +157,13 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 + themoviedb-season: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + example: 1 thetvdb: oneOf: - type: "null" @@ -129,13 +171,13 @@ $defs: minimum: 0 maximum: 50000000 example: 1337 - myanimelist: + thetvdb-season: oneOf: - type: "null" - type: integer minimum: 0 maximum: 50000000 - example: 1337 + example: 1 nullable_relation: oneOf: @@ -146,14 +188,19 @@ $defs: example: anidb: 1337 anilist: 1337 - anime-planet: spriggan + anime-planet: dororon-enma-kun + animecountdown: null + animenewsnetwork: null anisearch: null imdb: tt0164917 kitsu: null livechart: null + media: TV + myanimelist: null + themoviedb-season: 1 themoviedb: null + thetvdb-season: 1 thetvdb: null - myanimelist: null oneOf: - $ref: "#/$defs/nullable_relation" - type: array @@ -294,9 +341,11 @@ paths: schema: type: string enum: - - anilist - anidb + - anilist - anime-planet + - animecountdown + - animenewsnetwork - anisearch - kitsu - livechart @@ -370,12 +419,31 @@ paths: - type: string minLength: 1 maxLength: 50 + animecountdown: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + animenewsnetwork: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 anisearch: oneOf: - type: "null" - type: integer minimum: 0 maximum: 50000000 + imdb: + oneOf: + - type: "null" + - type: string + pattern: tt\d+ + minLength: 1 + maxLength: 50 kitsu: oneOf: - type: "null" @@ -388,12 +456,48 @@ paths: - type: integer minimum: 0 maximum: 50000000 + media: + oneOf: + - type: "null" + - type: string + minLength: 0 + maxLength: 10 myanimelist: oneOf: - type: "null" - type: integer minimum: 0 maximum: 50000000 + simkl: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + themoviedb: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + themoviedb-season: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + thetvdb: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + thetvdb-season: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 - type: array minItems: 1 maxItems: 100 @@ -420,12 +524,31 @@ paths: - type: string minLength: 1 maxLength: 50 + animecountdown: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + animenewsnetwork: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 anisearch: oneOf: - type: "null" - type: integer minimum: 0 maximum: 50000000 + imdb: + oneOf: + - type: "null" + - type: string + pattern: tt\d+ + minLength: 1 + maxLength: 50 kitsu: oneOf: - type: "null" @@ -438,12 +561,48 @@ paths: - type: integer minimum: 0 maximum: 50000000 + media: + oneOf: + - type: "null" + - type: string + minLength: 0 + maxLength: 10 myanimelist: oneOf: - type: "null" - type: integer minimum: 0 maximum: 50000000 + simkl: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + themoviedb: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + themoviedb-season: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + thetvdb: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 + thetvdb-season: + oneOf: + - type: "null" + - type: integer + minimum: 0 + maximum: 50000000 responses: "200": diff --git a/src/db/db.ts b/src/db/db.ts index e17aaf96a..50daa6b48 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -1,4 +1,4 @@ -import { mkdirSync } from "node:fs" +import { existsSync } from "node:fs" import { createDatabase } from "db0" import sqlite from "db0/connectors/node-sqlite" @@ -10,16 +10,30 @@ import { ActuallyWorkingMigrationProvider } from "./file-provider.ts" export const Source = { AniDB: "anidb", AniList: "anilist", + AnimeCountdown: "animecountdown", + AnimeNewsNetwork: "animenewsnetwork", AnimePlanet: "anime-planet", AniSearch: "anisearch", IMDB: "imdb", Kitsu: "kitsu", LiveChart: "livechart", + MAL: "myanimelist", + MediaType: "media", + Simkl: "simkl", TheMovieDB: "themoviedb", + TheMovieDBSeason: "themoviedb-season", TheTVDB: "thetvdb", - MAL: "myanimelist", + TheTVDBSeason: "thetvdb-season", } as const export type SourceValue = (typeof Source)[keyof typeof Source] +export const NonUniqueFields = [ + Source.IMDB, + Source.MediaType, + Source.TheMovieDB, + Source.TheMovieDBSeason, + Source.TheTVDB, + Source.TheTVDBSeason, +] as (keyof Relation)[] export type Relation = { [Source.AniDB]?: number @@ -29,9 +43,15 @@ export type Relation = { [Source.IMDB]?: `tt${string}` [Source.Kitsu]?: number [Source.LiveChart]?: number + [Source.AnimeNewsNetwork]?: number [Source.TheMovieDB]?: number [Source.TheTVDB]?: number [Source.MAL]?: number + [Source.TheTVDBSeason]?: number + [Source.TheMovieDBSeason]?: number + [Source.Simkl]?: number + [Source.AnimeCountdown]?: number + [Source.MediaType]?: string } export type OldRelation = Pick @@ -41,9 +61,6 @@ export interface Database { relations: Relation } -// Ensure SQLite directory exists -mkdirSync("./dir", { recursive: true }) - const sqliteDb = sqlite( process.env.VITEST_POOL_ID == null ? { path: `./db/${process.env.NODE_ENV ?? "development"}.sqlite3` } @@ -58,6 +75,6 @@ export const db = new Kysely({ export const migrator = new Migrator({ db, provider: new ActuallyWorkingMigrationProvider( - process.env.NODE_ENV !== "test" ? "dist/migrations" : "src/migrations", + existsSync("src/migrations") ? "src/migrations" : "dist/migrations", ), }) diff --git a/src/index.ts b/src/index.ts index b142c3471..478b75884 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { serve } from "h3" import { createApp } from "./app.ts" -import { config } from "./config.ts" +import { config, Environment } from "./config.ts" import { migrator } from "./db/db.ts" import { updateRelations } from "./update.ts" @@ -11,7 +11,7 @@ const { NODE_ENV, PORT } = config const runUpdateScript = async () => updateRelations() -if (NODE_ENV === "production") { +if (NODE_ENV === Environment.Prod) { void runUpdateScript() // eslint-disable-next-line ts/no-misused-promises diff --git a/src/migrations/20190611171759_initial.ts b/src/migrations/20190611171759_initial.ts index 6a5b372b8..ebd253553 100644 --- a/src/migrations/20190611171759_initial.ts +++ b/src/migrations/20190611171759_initial.ts @@ -1,6 +1,6 @@ import { type Kysely, sql } from "kysely" -export async function up(db: Kysely): Promise { +export async function up(db: Kysely): Promise { await sql`PRAGMA journal_mode=WAL`.execute(db) await db.schema diff --git a/src/migrations/20260124120530_schema.ts b/src/migrations/20260124120530_schema.ts new file mode 100644 index 000000000..2078c27c4 --- /dev/null +++ b/src/migrations/20260124120530_schema.ts @@ -0,0 +1,12 @@ +import type { Kysely } from "kysely" + +export async function up(db: Kysely): Promise { + await db.schema.alterTable("relations").addColumn("themoviedb-season", "integer").execute() + await db.schema.alterTable("relations").addColumn("thetvdb-season", "integer").execute() + // unique, but sqlite can't add unique columns to tables + await db.schema.alterTable("relations").addColumn("animenewsnetwork", "integer").execute() + // unique, but sqlite can't add unique columns to tables + await db.schema.alterTable("relations").addColumn("animecountdown", "integer").execute() + await db.schema.alterTable("relations").addColumn("simkl", "integer").execute() + await db.schema.alterTable("relations").addColumn("media", "text").execute() +} diff --git a/src/routes/v2/ids/handler.test.ts b/src/routes/v2/ids/handler.test.ts index 77df1ebb1..7e8e68253 100644 --- a/src/routes/v2/ids/handler.test.ts +++ b/src/routes/v2/ids/handler.test.ts @@ -12,13 +12,19 @@ const createRelations = async ( anidb: id++, anilist: id++, "anime-planet": `${id++}`, + animecountdown: id++, + animenewsnetwork: id++, anisearch: id++, imdb: `tt${id++}`, kitsu: id++, livechart: id++, + media: "TV", + myanimelist: id++, + simkl: id++, themoviedb: id++, + "themoviedb-season": 1, thetvdb: id++, - myanimelist: id++, + "thetvdb-season": 1, })) // Insert each relation @@ -75,13 +81,19 @@ describe("query params", () => { anidb: 1337, anilist: 1337, "anime-planet": null!, + animecountdown: null!, + animenewsnetwork: null!, anisearch: null!, imdb: null!, kitsu: null!, livechart: null!, + media: null!, + myanimelist: null!, + simkl: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, - myanimelist: null!, + "thetvdb-season": null!, } await db.insertInto("relations").values(relation).execute() @@ -159,13 +171,19 @@ describe("json body", () => { anidb: 1337, anilist: 1337, "anime-planet": null!, + animecountdown: null!, + animenewsnetwork: null!, anisearch: null!, imdb: null!, kitsu: null!, livechart: null!, + media: null!, + myanimelist: null!, + simkl: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, - myanimelist: null!, + "thetvdb-season": null!, } await db.insertInto("relations").values(relation).execute() diff --git a/src/routes/v2/ids/schemas/common.ts b/src/routes/v2/ids/schemas/common.ts index b7644aca7..8e43d1530 100644 --- a/src/routes/v2/ids/schemas/common.ts +++ b/src/routes/v2/ids/schemas/common.ts @@ -4,6 +4,8 @@ import * as v from "valibot" export const numberIdSourceSchema = v.picklist([ "anilist", "anidb", + "animecountdown", + "animenewsnetwork", "anisearch", "kitsu", "livechart", diff --git a/src/routes/v2/ids/schemas/json-body.test.ts b/src/routes/v2/ids/schemas/json-body.test.ts index 001943afe..7cade4294 100644 --- a/src/routes/v2/ids/schemas/json-body.test.ts +++ b/src/routes/v2/ids/schemas/json-body.test.ts @@ -19,6 +19,8 @@ const okCases = [ anidb: 1337, anilist: 1337, "anime-planet": "1337", + animecountdown: 1337, + animenewsnetwork: 1337, anisearch: 1337, kitsu: 1337, livechart: 1337, @@ -46,6 +48,7 @@ const badCases = [ [{ imdb: 1337 }, false], [{ themoviedb: 1337 }, false], [{ thetvdb: 1337 }, false], + [{ simkl: 1337 }, false], ] satisfies Cases const mapToSingularArrayInput = (cases: Cases): Cases => diff --git a/src/routes/v2/ids/schemas/json-body.ts b/src/routes/v2/ids/schemas/json-body.ts index c686debe1..e1bb2cc13 100644 --- a/src/routes/v2/ids/schemas/json-body.ts +++ b/src/routes/v2/ids/schemas/json-body.ts @@ -11,6 +11,8 @@ export const singularItemInputSchema = v.pipe( anidb: numberIdSchema, anilist: numberIdSchema, "anime-planet": stringIdSchema, + animecountdown: numberIdSchema, + animenewsnetwork: numberIdSchema, anisearch: numberIdSchema, kitsu: numberIdSchema, livechart: numberIdSchema, diff --git a/src/routes/v2/include.test-utils.ts b/src/routes/v2/include.test-utils.ts index f07b1a9dc..b97dcffa2 100644 --- a/src/routes/v2/include.test-utils.ts +++ b/src/routes/v2/include.test-utils.ts @@ -53,6 +53,26 @@ export const testIncludeQueryParam = ( expect(response.headers.get("content-type")).toContain("application/json") }) + test("duplicate sources (anilist,anilist,themoviedb)", async () => { + await db + .insertInto("relations") + .values({ anilist: 1337, thetvdb: 1337, themoviedb: 1337, imdb: "tt1337" }) + .execute() + + const query = new URLSearchParams({ + source, + id: prefixify(source, "1337"), + include: [Source.AniList, Source.AniList, Source.TheMovieDB].join(","), + }) + const response = await app.fetch(new Request(`http://localhost${path}?${query.toString()}`)) + + await expect(response.json()).resolves.toStrictEqual( + arrayify({ anilist: 1337, themoviedb: 1337 }), + ) + expect(response.status).toBe(200) + expect(response.headers.get("content-type")).toContain("application/json") + }) + test("all the sources", async () => { await db .insertInto("relations") @@ -70,13 +90,19 @@ export const testIncludeQueryParam = ( anidb: null, anilist: 1337, "anime-planet": null, + animecountdown: null, + animenewsnetwork: null, anisearch: null, imdb: null, kitsu: null, livechart: null, + media: null, + myanimelist: null, + simkl: null, themoviedb: null, + "themoviedb-season": null, thetvdb: null, - myanimelist: null, + "thetvdb-season": null, } expectedResult[source] = prefixify(source, 1337) as never diff --git a/src/routes/v2/include.ts b/src/routes/v2/include.ts index 58b21781e..b4d01ac03 100644 --- a/src/routes/v2/include.ts +++ b/src/routes/v2/include.ts @@ -2,28 +2,30 @@ import * as v from "valibot" import { db, Source, type SourceValue } from "../../db/db.ts" +const allSources = Object.values(Source) + export const includeSchema = v.object({ include: v.optional( v.pipe( v.string(), v.regex(/^[\-a-z,]+$/, "Invalid `include` query"), + v.transform((value) => value.split(",")), + v.transform((sources) => (sources.length > 1 ? [...new Set(sources)] : sources)), v.minLength(1), - v.maxLength(100), + v.maxLength(allSources.length), ), ), }) export type IncludeQuery = v.InferOutput -const sources = Object.values(Source) -const selectAll = sources.map((column) => db.dynamic.ref(column)) -export const buildSelectFromInclude = (include: string | null | undefined) => { +const selectAll = allSources.map((column) => db.dynamic.ref(column)) +export const buildSelectFromInclude = (include: string[] | null | undefined) => { if (include == null) { return selectAll } return include - .split(",") - .filter((inclusion) => sources.includes(inclusion as SourceValue)) + .filter((inclusion) => allSources.includes(inclusion as SourceValue)) .map((column) => db.dynamic.ref(column)) } diff --git a/src/routes/v2/special/handler.test.ts b/src/routes/v2/special/handler.test.ts index dfa67eea7..eb29b52e8 100644 --- a/src/routes/v2/special/handler.test.ts +++ b/src/routes/v2/special/handler.test.ts @@ -13,13 +13,19 @@ const createRelations = async ( anidb: id++, anilist: id++, "anime-planet": `${id++}`, + animecountdown: id++, + animenewsnetwork: id++, anisearch: id++, imdb: `tt${specialId ?? id++}`, kitsu: id++, livechart: id++, + media: "TV", + myanimelist: id++, + simkl: id++, themoviedb: specialId ?? id++, + "themoviedb-season": specialId ?? 1, thetvdb: specialId ?? id++, - myanimelist: id++, + "thetvdb-season": specialId ?? 1, })) await db.insertInto("relations").values(relations).execute() @@ -66,13 +72,19 @@ describe("imdb", () => { anidb: 1337, anilist: 1337, "anime-planet": null!, + animecountdown: null!, + animenewsnetwork: null!, anisearch: null!, imdb: "tt1337", kitsu: null!, livechart: null!, + media: null!, + myanimelist: null!, + simkl: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: null!, - myanimelist: null!, + "thetvdb-season": null!, } await db.insertInto("relations").values(relation).execute() @@ -111,13 +123,19 @@ describe("thetvdb", () => { anidb: 1337, anilist: 1337, "anime-planet": null!, + animecountdown: null!, + animenewsnetwork: null!, anisearch: null!, imdb: null!, kitsu: null!, livechart: null!, + media: null!, + myanimelist: null!, + simkl: null!, themoviedb: null!, + "themoviedb-season": null!, thetvdb: 1337, - myanimelist: null!, + "thetvdb-season": 1, } await db.insertInto("relations").values(relation).execute() @@ -158,13 +176,19 @@ describe("themoviedb", () => { anidb: 1337, anilist: 1337, "anime-planet": null!, + animecountdown: null!, + animenewsnetwork: null!, anisearch: null!, imdb: null!, kitsu: null!, livechart: null!, + media: null!, + myanimelist: null!, + simkl: null!, themoviedb: 1337, + "themoviedb-season": 1, thetvdb: null!, - myanimelist: null!, + "thetvdb-season": null!, } await db.insertInto("relations").values(relation).execute() diff --git a/src/update.test.ts b/src/update.test.ts index 55908428f..5c9726940 100644 --- a/src/update.test.ts +++ b/src/update.test.ts @@ -105,6 +105,7 @@ it("handles duplicates", async () => { Source.AniSearch, Source.Kitsu, Source.LiveChart, + Source.AnimeNewsNetwork, Source.MAL, ] diff --git a/src/update.ts b/src/update.ts index 0386547cb..19d751f93 100644 --- a/src/update.ts +++ b/src/update.ts @@ -2,7 +2,8 @@ import { log } from "evlog" import xior, { type XiorError } from "xior" import errorRetryPlugin from "xior/plugins/error-retry" -import { db, type Relation, Source, type SourceValue } from "./db/db.ts" +import { config, Environment } from "./config.ts" +import { db, migrator, NonUniqueFields, type Relation, Source, type SourceValue } from "./db/db.ts" import { updateBasedOnManualRules } from "./manual-rules.ts" const http = xior.create({ responseType: "json" }) @@ -12,6 +13,7 @@ const isXiorError = (response: T | XiorError): response is XiorError => "stack" in (response as XiorError) export type AnimeListsSchema = Array<{ + type?: string anidb_id?: number anilist_id?: number "anime-planet_id"?: string @@ -20,8 +22,15 @@ export type AnimeListsSchema = Array<{ kitsu_id?: number livechart_id?: number mal_id?: number + animenewsnetwork_id?: number + animecountdown_id?: number + simkl_id?: number themoviedb_id?: number | "unknown" tvdb_id?: number + season?: { + tvdb?: number + tmdb?: number + } }> const fetchDatabase = async (): Promise => { @@ -59,7 +68,7 @@ const handleBadValues = ( // Removes duplicate source-id pairs from the list, except for thetvdb and themoviedb ids export const removeDuplicates = (entries: Relation[]): Relation[] => { const sources = (Object.values(Source) as SourceValue[]).filter( - (source) => source !== Source.TheTVDB && source !== Source.TheMovieDB && source !== Source.IMDB, + (source) => !NonUniqueFields.includes(source), ) const existing = new Map>(sources.map((name) => [name, new Set()])) @@ -69,10 +78,7 @@ export const removeDuplicates = (entries: Relation[]): Relation[] => { // Ignore nulls if (id == null) continue - // Ignore sources with one-to-many relations - if (source === Source.TheTVDB || source === Source.TheMovieDB || source === Source.IMDB) { - continue - } + if (NonUniqueFields.includes(source)) continue if (existing.get(source)!.has(id)) return false @@ -89,13 +95,19 @@ export const formatEntry = (entry: AnimeListsSchema[number]): Relation => ({ anidb: handleBadValues(entry.anidb_id), anilist: handleBadValues(entry.anilist_id), "anime-planet": handleBadValues(entry["anime-planet_id"]), + animecountdown: handleBadValues(entry.animecountdown_id), + animenewsnetwork: handleBadValues(entry.animenewsnetwork_id), anisearch: handleBadValues(entry.anisearch_id), imdb: handleBadValues(entry.imdb_id), kitsu: handleBadValues(entry.kitsu_id), livechart: handleBadValues(entry.livechart_id), + media: handleBadValues(entry.type), myanimelist: handleBadValues(entry.mal_id), + simkl: handleBadValues(entry.simkl_id), themoviedb: handleBadValues(entry.themoviedb_id), + "themoviedb-season": handleBadValues(entry.season?.tmdb), thetvdb: handleBadValues(entry.tvdb_id), + "thetvdb-season": handleBadValues(entry.season?.tvdb), }) export const updateRelations = async () => { @@ -120,6 +132,13 @@ export const updateRelations = async () => { const goodEntries = removeDuplicates(formattedEntries) log.info("update", `Removed duplicates. ${goodEntries.length} remain.`) + if (config.NODE_ENV !== Environment.Prod) { + const { error } = await migrator.migrateToLatest() + if (error != null) { + throw new Error("Executing migrations failed.", { cause: error }) + } + } + log.info("update", "Updating database...") await db.transaction().execute(async (trx) => { // Delete all existing relations