feat: integrate typesense

.gitignore

@@ -4,3 +4,5 @@
 .env.test.local
 .env.production.local
 .env.local
+
+data/

docker-compose.yml

@@ -0,0 +1,24 @@
+version: '3.8'
+
+services:
+  redis:
+    image: redis:latest
+    ports:
+      - "6379:6379"
+    volumes:
+      - ./data/redis-data:/data
+
+  typesense:
+    image: typesense/typesense:0.24.1
+    restart: on-failure
+    ports:
+      - "8108:8108"
+    volumes:
+      - ./data/typesense-data:/data
+    env_file: .env
+    command: '--data-dir /data'
+
+volumes:
+  redis-data:
+  typesense-data:
+

fresh.gen.ts

@@ -20,16 +20,17 @@ import * as $14 from "./routes/api/movies/index.ts";
 import * as $15 from "./routes/api/query/index.ts";
 import * as $16 from "./routes/api/recipes/[name].ts";
 import * as $17 from "./routes/api/recipes/index.ts";
-import * as $18 from "./routes/api/tmdb/[id].ts";
-import * as $19 from "./routes/api/tmdb/credits/[id].ts";
-import * as $20 from "./routes/api/tmdb/query.ts";
-import * as $21 from "./routes/articles/[name].tsx";
-import * as $22 from "./routes/articles/index.tsx";
-import * as $23 from "./routes/index.tsx";
-import * as $24 from "./routes/movies/[name].tsx";
-import * as $25 from "./routes/movies/index.tsx";
-import * as $26 from "./routes/recipes/[name].tsx";
-import * as $27 from "./routes/recipes/index.tsx";
+import * as $18 from "./routes/api/resources.ts";
+import * as $19 from "./routes/api/tmdb/[id].ts";
+import * as $20 from "./routes/api/tmdb/credits/[id].ts";
+import * as $21 from "./routes/api/tmdb/query.ts";
+import * as $22 from "./routes/articles/[name].tsx";
+import * as $23 from "./routes/articles/index.tsx";
+import * as $24 from "./routes/index.tsx";
+import * as $25 from "./routes/movies/[name].tsx";
+import * as $26 from "./routes/movies/index.tsx";
+import * as $27 from "./routes/recipes/[name].tsx";
+import * as $28 from "./routes/recipes/index.tsx";
 import * as $$0 from "./islands/Counter.tsx";
 import * as $$1 from "./islands/IngredientsList.tsx";
 import * as $$2 from "./islands/KMenu.tsx";
@@ -59,16 +60,17 @@ const manifest = {
     "./routes/api/query/index.ts": $15,
     "./routes/api/recipes/[name].ts": $16,
     "./routes/api/recipes/index.ts": $17,
-    "./routes/api/tmdb/[id].ts": $18,
-    "./routes/api/tmdb/credits/[id].ts": $19,
-    "./routes/api/tmdb/query.ts": $20,
-    "./routes/articles/[name].tsx": $21,
-    "./routes/articles/index.tsx": $22,
-    "./routes/index.tsx": $23,
-    "./routes/movies/[name].tsx": $24,
-    "./routes/movies/index.tsx": $25,
-    "./routes/recipes/[name].tsx": $26,
-    "./routes/recipes/index.tsx": $27,
+    "./routes/api/resources.ts": $18,
+    "./routes/api/tmdb/[id].ts": $19,
+    "./routes/api/tmdb/credits/[id].ts": $20,
+    "./routes/api/tmdb/query.ts": $21,
+    "./routes/articles/[name].tsx": $22,
+    "./routes/articles/index.tsx": $23,
+    "./routes/index.tsx": $24,
+    "./routes/movies/[name].tsx": $25,
+    "./routes/movies/index.tsx": $26,
+    "./routes/recipes/[name].tsx": $27,
+    "./routes/recipes/index.tsx": $28,
   },
   islands: {
     "./islands/Counter.tsx": $$0,

lib/env.ts

@@ -16,3 +16,6 @@ const duration = Deno.env.get("SESSION_DURATION");
 export const SESSION_DURATION = duration ? +duration : (60 * 60 * 24);
 
 export const JWT_SECRET = Deno.env.get("JWT_SECRET");
+
+export const TYPESENSE_URL = Deno.env.get("TYPESENSE_URL")||"http://localhost:8108"
+export const TYPESENSE_API_KEY = Deno.env.get("TYPESENSE_API_KEY")

lib/resource/articles.ts

@@ -1,7 +1,7 @@
-import { parseDocument, renderMarkdown } from "@lib/documents.ts";
+import { parseDocument } from "@lib/documents.ts";
 import { parse } from "yaml";
 import { createCrud } from "@lib/crud.ts";
-import { stringify } from "https://deno.land/std@0.197.0/yaml/stringify.ts";
+import { stringify } from "$std/yaml/stringify.ts";
 import { extractHashTags, formatDate } from "@lib/string.ts";
 import { fixRenderedMarkdown } from "@lib/helpers.ts";
 
@@ -92,7 +92,7 @@ function parseArticle(original: string, id: string): Article {
     id,
     name,
     tags,
-    content: renderMarkdown(content),
+    content,
     meta,
   };
 }

lib/resource/movies.ts

@@ -90,7 +90,7 @@ export function parseMovie(original: string, id: string): Movie {
     id,
     name,
     tags,
-    description: renderMarkdown(description),
+    description,
     meta,
   };
 }

lib/resource/recipes.ts

@@ -178,9 +178,9 @@ export function parseRecipe(original: string, id: string): Recipe {
     meta,
     name,
     tags,
-    description: description ? renderMarkdown(description) : "",
+    description,
     ingredients,
-    preparation: preparation ? renderMarkdown(preparation) : "",
+    preparation,
   };
 }
 

lib/typesense.ts

@@ -0,0 +1,153 @@
+import { Client } from "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/mod.ts";
+import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts";
+import { getAllMovies } from "@lib/resource/movies.ts";
+import { getAllRecipes } from "@lib/resource/recipes.ts";
+import { getAllArticles } from "@lib/resource/articles.ts";
+
+function sanitizeStringForTypesense(input: string) {
+  // Remove backslashes
+  const withoutBackslashes = input.replace(/\\/g, "");
+
+  // Remove control characters other than carriage return and line feed
+  const withoutControlCharacters = withoutBackslashes.replace(
+    /[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g,
+    "",
+  );
+
+  // Remove Unicode characters above U+FFFF
+  const withoutUnicodeAboveFFFF = withoutControlCharacters.replace(
+    /[\uD800-\uDBFF][\uDC00-\uDFFF]/g,
+    "",
+  );
+
+  return withoutUnicodeAboveFFFF;
+}
+
+// Use a promise to initialize the client as needed, rather than at import time.
+let clientPromise: Promise<Client | null> | undefined;
+
+clientPromise?.then((client) => {
+  client?.collections().create({
+    name: "resources",
+    fields: [],
+  });
+});
+
+export function getTypeSenseClient(): Promise<Client | null> {
+  if (clientPromise === undefined) {
+    let typesenseUrl: URL;
+    try {
+      typesenseUrl = new URL(TYPESENSE_URL);
+    } catch (_err) {
+      return Promise.resolve(null);
+    }
+
+    clientPromise = new Promise((resolve) => {
+      if (!TYPESENSE_API_KEY) {
+        return resolve(null);
+      }
+      const client = new Client({
+        nodes: [{
+          host: typesenseUrl.hostname,
+          port: +typesenseUrl.port || 8108,
+          protocol: typesenseUrl.protocol.slice(0, -1),
+        }],
+        apiKey: TYPESENSE_API_KEY,
+        connectionTimeoutSeconds: 2,
+      });
+      resolve(client);
+    });
+  }
+  return clientPromise;
+}
+
+async function initializeTypesense() {
+  try {
+    const client = await getTypeSenseClient();
+    if (!client) return;
+    // Create the "resources" collection if it doesn't exist
+    const collections = await client.collections().retrieve();
+    const resourcesCollection = collections.find((collection) =>
+      collection.name === "resources"
+    );
+    if (!resourcesCollection) {
+      await client.collections().create({
+        name: "resources",
+        fields: [
+          { name: "name", type: "string" },
+          { name: "type", type: "string", facet: true },
+          { name: "date", type: "string", optional: true },
+          { name: "rating", type: "int32", facet: true },
+          { name: "description", type: "string", optional: true },
+        ],
+        default_sorting_field: "rating", // Default field for sorting
+      });
+      console.log('[typesense] created "resources" collection');
+    } else {
+      console.log('[typesense] collection "resources" already exists.');
+    }
+  } catch (error) {
+    console.error("[typesense] error initializing", error);
+  }
+}
+
+const init = initializeTypesense();
+
+async function synchronizeWithTypesense() {
+  await init;
+  try {
+    const allResources = (await Promise.all([
+      getAllMovies(),
+      getAllArticles(),
+      getAllRecipes(),
+    ])).flat(); // Replace with your function to get all resources from the database
+
+    const client = await getTypeSenseClient();
+    if (!client) return;
+
+    // Convert the list of documents to Typesense compatible format (array of objects)
+    const typesenseDocuments = allResources.map((resource) => {
+      return {
+        id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs
+        name: sanitizeStringForTypesense(resource.name),
+        description: sanitizeStringForTypesense(
+          resource?.description || resource?.content || "",
+        ),
+        rating: resource?.meta?.rating || 0,
+        date: resource?.meta?.date?.toString() || "",
+        type: resource.type,
+      };
+    });
+
+    await client.collections("resources").documents().import(
+      typesenseDocuments,
+      { action: "upsert" },
+    );
+
+    // Get all the IDs of documents currently indexed in Typesense
+    // const allTypesenseDocuments = await client.collections("resources")
+    //   .documents()
+    //   .search({ q: "*", query_by: "name" });
+
+    // const documentIds = allTypesenseDocuments.hits?.map((doc) => doc.id);
+
+    // Find deleted document IDs by comparing the Typesense document IDs with the current list of resources
+    // const deletedDocumentIds = documentIds?.filter((id) =>
+    //   !allResources.some((resource) => resource.id.toString() === id)
+    // );
+
+    // Delete the documents with IDs found in deletedDocumentIds
+    // await Promise.all(
+    //   deletedDocumentIds?.map((id) =>
+    //     client.collections("resources").documents()
+    //   ),
+    // );
+
+    console.log("Data synchronized with Typesense.");
+  } catch (error) {
+    console.error("Error synchronizing data with Typesense:", error);
+  }
+}
+
+// Call the synchronizeWithTypesense function to trigger the synchronization
+synchronizeWithTypesense();

routes/api/resources.ts

@@ -0,0 +1,31 @@
+import { Handlers } from "$fresh/server.ts";
+import { BadRequestError } from "@lib/errors.ts";
+import { getTypeSenseClient } from "@lib/typesense.ts";
+import { json } from "@lib/helpers.ts";
+
+export const handler: Handlers = {
+  async GET(req, _ctx) {
+    const url = new URL(req.url);
+    const query = url.searchParams.get("q");
+    if (!query) {
+      throw new BadRequestError('Query parameter "q" is required.');
+    }
+
+    const query_by = url.searchParams.get("query_by") || "name,description";
+
+    const typesenseClient = await getTypeSenseClient();
+    if (!typesenseClient) {
+      throw new Error("Query not available");
+    }
+
+    // Perform the Typesense search
+    const searchResults = await typesenseClient.collections("resources")
+      .documents().search({
+        q: query,
+        query_by,
+        limit_hits: 100,
+      });
+
+    return json(searchResults.hits);
+  },
+};