import { Client } from "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/mod.ts"; import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts"; import { getAllMovies } from "@lib/resource/movies.ts"; import { getAllRecipes } from "@lib/resource/recipes.ts"; import { getAllArticles } from "@lib/resource/articles.ts"; import { createLogger } from "@lib/log.ts"; const log = createLogger("typesense"); function sanitizeStringForTypesense(input: string) { // Remove backslashes const withoutBackslashes = input.replace(/\\/g, ""); // Remove control characters other than carriage return and line feed const withoutControlCharacters = withoutBackslashes.replace( /[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g, "", ); // Remove Unicode characters above U+FFFF const withoutUnicodeAboveFFFF = withoutControlCharacters.replace( /[\uD800-\uDBFF][\uDC00-\uDFFF]/g, "", ); return withoutUnicodeAboveFFFF; } // Use a promise to initialize the client as needed, rather than at import time. let clientPromise: Promise | undefined; export function getTypeSenseClient(): Promise { if (clientPromise === undefined) { let typesenseUrl: URL; try { typesenseUrl = new URL(TYPESENSE_URL); } catch (_err) { return Promise.resolve(null); } clientPromise = new Promise((resolve) => { if (!TYPESENSE_API_KEY) { return resolve(null); } const client = new Client({ nodes: [{ host: typesenseUrl.hostname, port: +typesenseUrl.port || 8108, protocol: typesenseUrl.protocol.slice(0, -1), }], apiKey: TYPESENSE_API_KEY, connectionTimeoutSeconds: 2, }); resolve(client); }); } return clientPromise; } async function initializeTypesense() { try { const client = await getTypeSenseClient(); if (!client) return; // Create the "resources" collection if it doesn't exist const collections = await client.collections().retrieve(); const resourcesCollection = collections.find((collection) => collection.name === "resources" ); if (!resourcesCollection) { await client.collections().create({ name: "resources", fields: [ { name: "name", type: "string" }, { name: "type", type: "string", facet: true }, { name: "date", type: "string", optional: true }, { name: "rating", type: "int32", facet: true }, { name: "tags", type: "string[]", facet: true }, { name: "description", type: "string", optional: true }, { name: "image", type: "string", optional: true }, ], default_sorting_field: "rating", // Default field for sorting }); log.info('created "resources" collection'); } else { log.info('collection "resources" already exists.'); } } catch (error) { log.error("error initializing", error); } } const init = initializeTypesense(); async function synchronizeWithTypesense() { await init; try { const allResources = (await Promise.all([ getAllMovies(), getAllArticles(), getAllRecipes(), ])).flat(); // Replace with your function to get all resources from the database const client = await getTypeSenseClient(); if (!client) return; // Convert the list of documents to Typesense compatible format (array of objects) const typesenseDocuments = allResources.map((resource) => { return { id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs name: sanitizeStringForTypesense(resource.name), description: sanitizeStringForTypesense( resource?.description || resource?.content || "", ), image: resource?.meta?.image, tags: resource?.tags || [], rating: resource?.meta?.rating || 0, date: resource?.meta?.date?.toString() || "", type: resource.type, }; }); await client.collections("resources").documents().import( typesenseDocuments, { action: "upsert" }, ); // Get all the IDs of documents currently indexed in Typesense const allTypesenseDocuments = await client.collections("resources") .documents().search({ q: "*", query_by: "name,type,date,description", per_page: 250, limit_hits: 9999, }); const documentIds = allTypesenseDocuments.hits?.map((doc) => doc?.document?.id ) as string[]; // Find deleted document IDs by comparing the Typesense document IDs with the current list of resources const deletedDocumentIds = documentIds?.filter((id) => !allResources.some((resource) => resource.id.toString() === id) ); // Delete the documents with IDs found in deletedDocumentIds await Promise.all( deletedDocumentIds?.map((id) => client.collections("resources").documents(id).delete() ), ); log.info("data synchronized"); } catch (error) { log.error("error synchronizing", error); } } // Call the synchronizeWithTypesense function to trigger the synchronization synchronizeWithTypesense();