memorium/lib/typesense.ts

import { Client } from "typesense";
import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts";
import { getAllMovies } from "@lib/resource/movies.ts";
import { getAllRecipes } from "@lib/resource/recipes.ts";
import { getAllArticles } from "@lib/resource/articles.ts";
import { createLogger } from "@lib/log.ts";
import { debounce } from "https://deno.land/std@0.193.0/async/mod.ts";

const log = createLogger("typesense");

function sanitizeStringForTypesense(input: string) {
  // Remove backslashes
  const withoutBackslashes = input.replace(/\\/g, "");

  // Remove control characters other than carriage return and line feed
  const withoutControlCharacters = withoutBackslashes.replace(
    /[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g,
    "",
  );

  // Remove Unicode characters above U+FFFF
  const withoutUnicodeAboveFFFF = withoutControlCharacters.replace(
    /[\uD800-\uDBFF][\uDC00-\uDFFF]/g,
    "",
  );

  return withoutUnicodeAboveFFFF;
}

// Use a promise to initialize the client as needed, rather than at import time.
let clientPromise: Promise<Client | null> | undefined;

export function getTypeSenseClient(): Promise<Client | null> {
  if (clientPromise === undefined) {
    let typesenseUrl: URL;
    try {
      typesenseUrl = new URL(TYPESENSE_URL);
    } catch (_err) {
      return Promise.resolve(null);
    }

    clientPromise = new Promise((resolve) => {
      if (!TYPESENSE_API_KEY) {
        return resolve(null);
      }
      const client = new Client({
        nodes: [{
          host: typesenseUrl.hostname,
          port: +typesenseUrl.port || 8108,
          protocol: typesenseUrl.protocol.slice(0, -1),
        }],
        apiKey: TYPESENSE_API_KEY,
        connectionTimeoutSeconds: 2,
      });
      resolve(client);
    });
  }
  return clientPromise;
}

async function initializeTypesense() {
  try {
    const client = await getTypeSenseClient();
    if (!client) return;
    // Create the "resources" collection if it doesn't exist
    const collections = await client.collections().retrieve();
    const resourcesCollection = collections.find((collection) =>
      collection.name === "resources"
    );
    if (!resourcesCollection) {
      await client.collections().create({
        name: "resources",
        fields: [
          { name: "name", type: "string" },
          { name: "type", type: "string", facet: true },
          { name: "date", type: "string", optional: true },
          { name: "author", type: "string", facet: true },
          { name: "rating", type: "int32", facet: true },
          { name: "tags", type: "string[]", facet: true },
          { name: "description", type: "string", optional: true },
          { name: "image", type: "string", optional: true },
        ],
        default_sorting_field: "rating", // Default field for sorting
      });
      log.info('created "resources" collection');
    } else {
      log.info('collection "resources" already exists.');
    }
  } catch (error) {
    log.error("error initializing", error);
  }
}

const init = initializeTypesense();

async function synchronizeWithTypesense() {
  await init;
  try {
    const allResources = (await Promise.all([
      getAllMovies(),
      getAllArticles(),
      getAllRecipes(),
    ])).flat(); // Replace with your function to get all resources from the database

    const client = await getTypeSenseClient();
    if (!client) return;

    // Convert the list of documents to Typesense compatible format (array of objects)
    const typesenseDocuments = allResources.map((resource) => {
      return {
        id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs
        name: sanitizeStringForTypesense(resource.name),
        description: sanitizeStringForTypesense(
          resource?.description || resource?.content || "",
        ),
        author: resource.meta?.author,
        image: resource.meta?.image,
        tags: resource?.tags || [],
        rating: resource.meta?.rating || 0,
        date: resource.meta?.date?.toString() || "",
        type: resource.type,
      };
    });

    await client.collections("resources").documents().import(
      typesenseDocuments,
      { action: "upsert" },
    );

    // Get all the IDs of documents currently indexed in Typesense
    const allTypesenseDocuments = await client.collections("resources")
      .documents().search({
        q: "*",
        query_by: "name,type,date,description",
        per_page: 250,
        limit_hits: 9999,
      });

    const deletedDocumentIds = allTypesenseDocuments.hits
      ?.map((doc) => doc?.document?.id)
      ?.filter((id) =>
        // Find deleted document IDs by comparing the Typesense document IDs with the current list of resources
        !allResources.some((resource) => resource.id.toString() === id)
      ).map((id) => client.collections("resources").documents(id).delete());

    // Delete the documents with IDs found in deletedDocumentIds
    if (deletedDocumentIds) {
      await Promise.all(
        deletedDocumentIds,
      );
    }

    log.info("data synchronized");
  } catch (error) {
    log.error("error synchronizing", error);
  }
}

// Call the synchronizeWithTypesense function to trigger the synchronization
synchronizeWithTypesense();

export const synchronize = debounce(synchronizeWithTypesense, 1000 * 60 * 5);