diff --git a/.gitignore b/.gitignore index 4e06ffc..328e180 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .env.test.local .env.production.local .env.local + +data/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3882edc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3.8' + +services: + redis: + image: redis:latest + ports: + - "6379:6379" + volumes: + - ./data/redis-data:/data + + typesense: + image: typesense/typesense:0.24.1 + restart: on-failure + ports: + - "8108:8108" + volumes: + - ./data/typesense-data:/data + env_file: .env + command: '--data-dir /data' + +volumes: + redis-data: + typesense-data: + diff --git a/fresh.gen.ts b/fresh.gen.ts index 6b12e21..3c8df59 100644 --- a/fresh.gen.ts +++ b/fresh.gen.ts @@ -20,16 +20,17 @@ import * as $14 from "./routes/api/movies/index.ts"; import * as $15 from "./routes/api/query/index.ts"; import * as $16 from "./routes/api/recipes/[name].ts"; import * as $17 from "./routes/api/recipes/index.ts"; -import * as $18 from "./routes/api/tmdb/[id].ts"; -import * as $19 from "./routes/api/tmdb/credits/[id].ts"; -import * as $20 from "./routes/api/tmdb/query.ts"; -import * as $21 from "./routes/articles/[name].tsx"; -import * as $22 from "./routes/articles/index.tsx"; -import * as $23 from "./routes/index.tsx"; -import * as $24 from "./routes/movies/[name].tsx"; -import * as $25 from "./routes/movies/index.tsx"; -import * as $26 from "./routes/recipes/[name].tsx"; -import * as $27 from "./routes/recipes/index.tsx"; +import * as $18 from "./routes/api/resources.ts"; +import * as $19 from "./routes/api/tmdb/[id].ts"; +import * as $20 from "./routes/api/tmdb/credits/[id].ts"; +import * as $21 from "./routes/api/tmdb/query.ts"; +import * as $22 from "./routes/articles/[name].tsx"; +import * as $23 from "./routes/articles/index.tsx"; +import * as $24 from "./routes/index.tsx"; +import * as $25 from "./routes/movies/[name].tsx"; +import * as $26 from "./routes/movies/index.tsx"; +import * as $27 from "./routes/recipes/[name].tsx"; +import * as $28 from "./routes/recipes/index.tsx"; import * as $$0 from "./islands/Counter.tsx"; import * as $$1 from "./islands/IngredientsList.tsx"; import * as $$2 from "./islands/KMenu.tsx"; @@ -59,16 +60,17 @@ const manifest = { "./routes/api/query/index.ts": $15, "./routes/api/recipes/[name].ts": $16, "./routes/api/recipes/index.ts": $17, - "./routes/api/tmdb/[id].ts": $18, - "./routes/api/tmdb/credits/[id].ts": $19, - "./routes/api/tmdb/query.ts": $20, - "./routes/articles/[name].tsx": $21, - "./routes/articles/index.tsx": $22, - "./routes/index.tsx": $23, - "./routes/movies/[name].tsx": $24, - "./routes/movies/index.tsx": $25, - "./routes/recipes/[name].tsx": $26, - "./routes/recipes/index.tsx": $27, + "./routes/api/resources.ts": $18, + "./routes/api/tmdb/[id].ts": $19, + "./routes/api/tmdb/credits/[id].ts": $20, + "./routes/api/tmdb/query.ts": $21, + "./routes/articles/[name].tsx": $22, + "./routes/articles/index.tsx": $23, + "./routes/index.tsx": $24, + "./routes/movies/[name].tsx": $25, + "./routes/movies/index.tsx": $26, + "./routes/recipes/[name].tsx": $27, + "./routes/recipes/index.tsx": $28, }, islands: { "./islands/Counter.tsx": $$0, diff --git a/lib/env.ts b/lib/env.ts index de3d20d..66de943 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -16,3 +16,6 @@ const duration = Deno.env.get("SESSION_DURATION"); export const SESSION_DURATION = duration ? +duration : (60 * 60 * 24); export const JWT_SECRET = Deno.env.get("JWT_SECRET"); + +export const TYPESENSE_URL = Deno.env.get("TYPESENSE_URL")||"http://localhost:8108" +export const TYPESENSE_API_KEY = Deno.env.get("TYPESENSE_API_KEY") diff --git a/lib/resource/articles.ts b/lib/resource/articles.ts index a20f697..bd2bf54 100644 --- a/lib/resource/articles.ts +++ b/lib/resource/articles.ts @@ -1,7 +1,7 @@ -import { parseDocument, renderMarkdown } from "@lib/documents.ts"; +import { parseDocument } from "@lib/documents.ts"; import { parse } from "yaml"; import { createCrud } from "@lib/crud.ts"; -import { stringify } from "https://deno.land/std@0.197.0/yaml/stringify.ts"; +import { stringify } from "$std/yaml/stringify.ts"; import { extractHashTags, formatDate } from "@lib/string.ts"; import { fixRenderedMarkdown } from "@lib/helpers.ts"; @@ -92,7 +92,7 @@ function parseArticle(original: string, id: string): Article { id, name, tags, - content: renderMarkdown(content), + content, meta, }; } diff --git a/lib/resource/movies.ts b/lib/resource/movies.ts index a85e580..c71434d 100644 --- a/lib/resource/movies.ts +++ b/lib/resource/movies.ts @@ -90,7 +90,7 @@ export function parseMovie(original: string, id: string): Movie { id, name, tags, - description: renderMarkdown(description), + description, meta, }; } diff --git a/lib/resource/recipes.ts b/lib/resource/recipes.ts index 341a4de..62ddfe6 100644 --- a/lib/resource/recipes.ts +++ b/lib/resource/recipes.ts @@ -178,9 +178,9 @@ export function parseRecipe(original: string, id: string): Recipe { meta, name, tags, - description: description ? renderMarkdown(description) : "", + description, ingredients, - preparation: preparation ? renderMarkdown(preparation) : "", + preparation, }; } diff --git a/lib/typesense.ts b/lib/typesense.ts new file mode 100644 index 0000000..65c370e --- /dev/null +++ b/lib/typesense.ts @@ -0,0 +1,153 @@ +import { Client } from "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/mod.ts"; +import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts"; +import { getAllMovies } from "@lib/resource/movies.ts"; +import { getAllRecipes } from "@lib/resource/recipes.ts"; +import { getAllArticles } from "@lib/resource/articles.ts"; + +function sanitizeStringForTypesense(input: string) { + // Remove backslashes + const withoutBackslashes = input.replace(/\\/g, ""); + + // Remove control characters other than carriage return and line feed + const withoutControlCharacters = withoutBackslashes.replace( + /[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g, + "", + ); + + // Remove Unicode characters above U+FFFF + const withoutUnicodeAboveFFFF = withoutControlCharacters.replace( + /[\uD800-\uDBFF][\uDC00-\uDFFF]/g, + "", + ); + + return withoutUnicodeAboveFFFF; +} + +// Use a promise to initialize the client as needed, rather than at import time. +let clientPromise: Promise | undefined; + +clientPromise?.then((client) => { + client?.collections().create({ + name: "resources", + fields: [], + }); +}); + +export function getTypeSenseClient(): Promise { + if (clientPromise === undefined) { + let typesenseUrl: URL; + try { + typesenseUrl = new URL(TYPESENSE_URL); + } catch (_err) { + return Promise.resolve(null); + } + + clientPromise = new Promise((resolve) => { + if (!TYPESENSE_API_KEY) { + return resolve(null); + } + const client = new Client({ + nodes: [{ + host: typesenseUrl.hostname, + port: +typesenseUrl.port || 8108, + protocol: typesenseUrl.protocol.slice(0, -1), + }], + apiKey: TYPESENSE_API_KEY, + connectionTimeoutSeconds: 2, + }); + resolve(client); + }); + } + return clientPromise; +} + +async function initializeTypesense() { + try { + const client = await getTypeSenseClient(); + if (!client) return; + // Create the "resources" collection if it doesn't exist + const collections = await client.collections().retrieve(); + const resourcesCollection = collections.find((collection) => + collection.name === "resources" + ); + if (!resourcesCollection) { + await client.collections().create({ + name: "resources", + fields: [ + { name: "name", type: "string" }, + { name: "type", type: "string", facet: true }, + { name: "date", type: "string", optional: true }, + { name: "rating", type: "int32", facet: true }, + { name: "description", type: "string", optional: true }, + ], + default_sorting_field: "rating", // Default field for sorting + }); + console.log('[typesense] created "resources" collection'); + } else { + console.log('[typesense] collection "resources" already exists.'); + } + } catch (error) { + console.error("[typesense] error initializing", error); + } +} + +const init = initializeTypesense(); + +async function synchronizeWithTypesense() { + await init; + try { + const allResources = (await Promise.all([ + getAllMovies(), + getAllArticles(), + getAllRecipes(), + ])).flat(); // Replace with your function to get all resources from the database + + const client = await getTypeSenseClient(); + if (!client) return; + + // Convert the list of documents to Typesense compatible format (array of objects) + const typesenseDocuments = allResources.map((resource) => { + return { + id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs + name: sanitizeStringForTypesense(resource.name), + description: sanitizeStringForTypesense( + resource?.description || resource?.content || "", + ), + rating: resource?.meta?.rating || 0, + date: resource?.meta?.date?.toString() || "", + type: resource.type, + }; + }); + + await client.collections("resources").documents().import( + typesenseDocuments, + { action: "upsert" }, + ); + + // Get all the IDs of documents currently indexed in Typesense + // const allTypesenseDocuments = await client.collections("resources") + // .documents() + // .search({ q: "*", query_by: "name" }); + + // const documentIds = allTypesenseDocuments.hits?.map((doc) => doc.id); + + // Find deleted document IDs by comparing the Typesense document IDs with the current list of resources + // const deletedDocumentIds = documentIds?.filter((id) => + // !allResources.some((resource) => resource.id.toString() === id) + // ); + + // Delete the documents with IDs found in deletedDocumentIds + // await Promise.all( + // deletedDocumentIds?.map((id) => + // client.collections("resources").documents() + // ), + // ); + + console.log("Data synchronized with Typesense."); + } catch (error) { + console.error("Error synchronizing data with Typesense:", error); + } +} + +// Call the synchronizeWithTypesense function to trigger the synchronization +synchronizeWithTypesense(); diff --git a/routes/api/resources.ts b/routes/api/resources.ts new file mode 100644 index 0000000..5dc79aa --- /dev/null +++ b/routes/api/resources.ts @@ -0,0 +1,31 @@ +import { Handlers } from "$fresh/server.ts"; +import { BadRequestError } from "@lib/errors.ts"; +import { getTypeSenseClient } from "@lib/typesense.ts"; +import { json } from "@lib/helpers.ts"; + +export const handler: Handlers = { + async GET(req, _ctx) { + const url = new URL(req.url); + const query = url.searchParams.get("q"); + if (!query) { + throw new BadRequestError('Query parameter "q" is required.'); + } + + const query_by = url.searchParams.get("query_by") || "name,description"; + + const typesenseClient = await getTypeSenseClient(); + if (!typesenseClient) { + throw new Error("Query not available"); + } + + // Perform the Typesense search + const searchResults = await typesenseClient.collections("resources") + .documents().search({ + q: query, + query_by, + limit_hits: 100, + }); + + return json(searchResults.hits); + }, +};