max/memorium
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: remove typesense

Browse Source
This commit is contained in:
max_richter
2025-01-05 23:14:19 +01:00
parent d0d49b217d
commit 709fb2d7be
21 changed files with 128 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/crud.ts
View File

@ -82,7 +82,7 @@ export function createCrud<T extends GenericResource>(
return addThumbnailToResource(res);
}
return res;
return { ...res, content };
}
function create(id: string, content: string | ArrayBuffer | T) {
const path = pathFromId(id);

3
lib/documents.ts
View File

@ -11,7 +11,6 @@ import remarkFrontmatter, {
import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts";
import { createLogger } from "@lib/log.ts";
import * as typesense from "@lib/typesense.ts";
import { db } from "@lib/sqlite/sqlite.ts";
import { documentTable } from "@lib/sqlite/schema.ts";
import { eq } from "drizzle-orm/sql";
@ -59,8 +58,6 @@ export function createDocument(
log.info("creating document", { name });
typesense.synchronize();
return fetch(SILVERBULLET_SERVER + "/" + name, {
body: content,
method: "PUT",

6
lib/env.ts
View File

@ -22,13 +22,9 @@ export const SESSION_DURATION = duration ? +duration : (60 * 60 * 24);
export const JWT_SECRET = Deno.env.get("JWT_SECRET");
export const TYPESENSE_URL = Deno.env.get("TYPESENSE_URL") ||
"http://localhost:8108";
export const TYPESENSE_API_KEY = Deno.env.get("TYPESENSE_API_KEY");
export const DATA_DIR = Deno.env.has("DATA_DIR")
? path.resolve(Deno.env.get("DATA_DIR")!)
: path.resolve(Deno.cwd(), "data");
export const LOG_LEVEL: string = Deno.env.get("LOG_LEVEL") ||
"debug";
"warn";

99
lib/search.ts
View File

@ -1,17 +1,20 @@
import { resources } from "@lib/resources.ts";
import { SearchResult } from "@lib/types.ts";
import { getTypeSenseClient } from "@lib/typesense.ts";
import fuzzysort from "npm:fuzzysort";
import { GenericResource } from "@lib/types.ts";
import { extractHashTags } from "@lib/string.ts";
import { getAllMovies, Movie } from "@lib/resource/movies.ts";
import { Article, getAllArticles } from "@lib/resource/articles.ts";
import { getAllRecipes, Recipe } from "@lib/resource/recipes.ts";
import { getAllSeries, Series } from "@lib/resource/series.ts";
type ResourceType = keyof typeof resources;
type SearchParams = {
q: string;
type?: ResourceType;
types?: string[];
tags?: string[];
rating?: string;
author?: string;
query_by?: string;
rating?: number;
authors?: string[];
};
export function parseResourceUrl(_url: string | URL): SearchParams | undefined {
@ -31,56 +34,60 @@ export function parseResourceUrl(_url: string | URL): SearchParams | undefined {
return {
q: query,
type: url.searchParams.get("type") as ResourceType || undefined,
types: url.searchParams.get("type")?.split(",") as ResourceType[] ||
undefined,
tags: hashTags,
rating: url.searchParams.get("rating") || undefined,
query_by: url.searchParams.get("query_by") || undefined,
rating: url.searchParams.has("rating")
? parseInt(url.searchParams.get("rating")!)
: undefined,
};
} catch (_err) {
return undefined;
}
}
const isResource = (
item: Movie | Series | Article | Recipe | boolean,
): item is Movie | Series | Article | Recipe => {
return !!item;
};
export async function searchResource(
{ q, query_by = "name,description,author,tags", tags = [], type, rating }:
SearchParams,
): Promise<SearchResult> {
const typesenseClient = await getTypeSenseClient();
if (!typesenseClient) {
throw new Error("Query not available");
}
{ q, tags = [], types, authors, rating }: SearchParams,
): Promise<GenericResource[]> {
console.log("searchResource", { q, tags, types, authors, rating });
const filter_by: string[] = [];
if (type) {
filter_by.push(`type:=${type}`);
}
let resources = (await Promise.all([
(!types || types.includes("movie")) && getAllMovies(),
(!types || types.includes("series")) && getAllSeries(),
(!types || types.includes("article")) && getAllArticles(),
(!types || types.includes("recipe")) && getAllRecipes(),
])).flat().filter(isResource);
if (tags?.length) {
filter_by.push(`tags:[${tags.map((t) => `\`${t}\``).join(",")}]`);
for (const tag of tags) {
q = q.replaceAll(`#${tag}`, "");
}
if (!q.trim().length) {
q = "*";
}
}
if (typeof rating !== "undefined") {
if (rating === "null") {
filter_by.push(`rating: null`);
} else {
filter_by.push(`rating: ${rating}`);
}
}
return await typesenseClient.collections("resources")
.documents().search({
q,
query_by,
facet_by: "rating,author,tags",
max_facet_values: 10,
filter_by: filter_by.join(" && "),
per_page: 50,
resources = resources.filter((r) => {
return tags?.every((t) => r.tags.includes(t));
});
}
if (authors?.length) {
resources = resources.filter((r) => {
return r?.meta?.author && authors.includes(r?.meta?.author);
});
}
if (rating) {
resources = resources.filter((r) => {
return r?.meta?.rating && r.meta.rating >= rating;
});
}
if (q.length && q !== "*") {
const results = fuzzysort.go(q, resources, {
keys: ["content", "name", "description"],
threshold: 0.3,
});
resources = results.map((r) => r.obj);
}
return resources;
}

6
lib/types.ts
View File

@ -1,4 +1,3 @@
import { SearchResponse } from "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/src/Typesense/Documents.ts";
import { resources } from "@lib/resources.ts";
export interface TMDBMovie {
@ -39,6 +38,7 @@ export type GenericResource = {
id: string;
tags?: string[];
type: keyof typeof resources;
content?: string;
meta?: {
image?: string;
author?: string;
@ -58,7 +58,7 @@ export interface GiteaOauthUser {
groups: any;
}
export type TypesenseDocument = {
export type SearchResult = {
id: string;
name: string;
type: keyof typeof resources;
@ -68,5 +68,3 @@ export type TypesenseDocument = {
description?: string;
image?: string;
};
export type SearchResult = SearchResponse<TypesenseDocument>;

178
lib/typesense.ts
View File

@ -1,178 +0,0 @@
import { Client } from "typesense";
import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts";
import { getAllMovies } from "@lib/resource/movies.ts";
import { getAllRecipes } from "@lib/resource/recipes.ts";
import { getAllArticles } from "@lib/resource/articles.ts";
import { createLogger } from "@lib/log.ts";
import { debounce } from "https://deno.land/std@0.193.0/async/mod.ts";
import { getAllSeries } from "@lib/resource/series.ts";
import { TypesenseDocument } from "@lib/types.ts";
const log = createLogger("typesense");
function sanitizeStringForTypesense(input: string) {
// Remove backslashes
const withoutBackslashes = input.replace(/\\/g, "");
// Remove control characters other than carriage return and line feed
const withoutControlCharacters = withoutBackslashes.replace(
/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g,
"",
);
// Remove Unicode characters above U+FFFF
const withoutUnicodeAboveFFFF = withoutControlCharacters.replace(
/[\uD800-\uDBFF][\uDC00-\uDFFF]/g,
"",
);
return withoutUnicodeAboveFFFF;
}
// Use a promise to initialize the client as needed, rather than at import time.
let clientPromise: Promise<Client | null> | undefined;
export function getTypeSenseClient(): Promise<Client | null> {
if (clientPromise === undefined) {
let typesenseUrl: URL;
try {
typesenseUrl = new URL(TYPESENSE_URL);
} catch (_err) {
return Promise.resolve(null);
}
clientPromise = new Promise((resolve) => {
if (!TYPESENSE_API_KEY) {
return resolve(null);
}
const client = new Client({
nodes: [{
host: typesenseUrl.hostname,
port: +typesenseUrl.port || 8108,
protocol: typesenseUrl.protocol.slice(0, -1),
}],
apiKey: TYPESENSE_API_KEY,
connectionTimeoutSeconds: 2,
});
resolve(client);
});
}
return clientPromise;
}
async function initializeTypesense() {
try {
const client = await getTypeSenseClient();
if (!client) return;
// Create the "resources" collection if it doesn't exist
const collections = await client.collections().retrieve();
const resourcesCollection = collections.find((collection) =>
collection.name === "resources"
);
if (!resourcesCollection) {
await client.collections().create({
name: "resources",
fields: [
{ name: "name", type: "string" },
{ name: "type", type: "string", facet: true },
{ name: "date", type: "string", optional: true },
{ name: "author", type: "string", facet: true, optional: true },
{ name: "rating", type: "int32", facet: true },
{ name: "tags", type: "string[]", facet: true },
{ name: "description", type: "string", optional: true },
{ name: "image", type: "string", optional: true },
],
default_sorting_field: "rating", // Default field for sorting
});
log.info('created "resources" collection');
} else {
log.info('collection "resources" already exists.');
}
} catch (error) {
log.error("error initializing", error);
}
}
const init = initializeTypesense();
export async function createTypesenseDocument(doc: TypesenseDocument) {
const client = await getTypeSenseClient();
if (!client) return;
// await client.collections("resources").documents().create(
// doc,
// { action: "upsert" },
// );
}
async function synchronizeWithTypesense() {
return;
await init;
try {
const allResources = (await Promise.all([
getAllMovies(),
getAllArticles(),
getAllRecipes(),
getAllSeries(),
])).flat(); // Replace with your function to get all resources from the database
const client = await getTypeSenseClient();
if (!client) return;
// Convert the list of documents to Typesense compatible format (array of objects)
const typesenseDocuments = allResources.map((resource) => {
return {
id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs
name: sanitizeStringForTypesense(resource.name),
description: sanitizeStringForTypesense(
resource?.description || resource?.content || "",
),
author: resource.meta?.author,
image: resource.meta?.image,
tags: resource?.tags || [],
rating: resource.meta?.rating || 0,
date: resource.meta?.date?.toString() || "",
type: resource.type,
};
});
return;
await client.collections("resources").documents().import(
typesenseDocuments,
{ action: "upsert" },
);
// Get all the IDs of documents currently indexed in Typesense
const allTypesenseDocuments = await client.collections("resources")
.documents().search({
q: "*",
query_by: "name,type,date,description",
per_page: 250,
limit_hits: 9999,
});
const deletedDocumentIds = allTypesenseDocuments.hits
?.map((doc) => doc?.document?.id)
?.filter((id) =>
// Find deleted document IDs by comparing the Typesense document IDs with the current list of resources
!allResources.some((resource) => resource.id.toString() === id)
).map((id) => client.collections("resources").documents(id).delete());
// Delete the documents with IDs found in deletedDocumentIds
if (deletedDocumentIds) {
await Promise.all(
deletedDocumentIds,
);
}
log.info("data synchronized");
} catch (error) {
log.error("error synchronizing", error);
}
}
// Call the synchronizeWithTypesense function to trigger the synchronization
synchronizeWithTypesense();
export const synchronize = debounce(synchronizeWithTypesense, 1000 * 60 * 5);