feat: remove typesense

This commit is contained in:
max_richter 2025-01-05 23:14:19 +01:00
parent d0d49b217d
commit 709fb2d7be
21 changed files with 128 additions and 381 deletions

View File

@ -65,7 +65,9 @@ const Image = (
style={props.style} style={props.style}
srcset={responsiveAttributes.srcset} srcset={responsiveAttributes.srcset}
sizes={responsiveAttributes.sizes} sizes={responsiveAttributes.sizes}
src={`/api/images?image=${asset(props.src)}`} src={`/api/images?image=${asset(props.src)}${
props.width ? `&width=${props.width}` : ""
}${props.height ? `&height=${props.height}` : ""}`}
width={props.width} width={props.width}
height={props.height} height={props.height}
class={props.class} class={props.class}

View File

@ -1,6 +1,6 @@
import { ComponentChildren } from "preact"; import { ComponentChildren } from "preact";
import Search from "@islands/Search.tsx"; import Search from "@islands/Search.tsx";
import { SearchResult } from "@lib/types.ts"; import { GenericResource, SearchResult } from "@lib/types.ts";
export type Props = { export type Props = {
children: ComponentChildren; children: ComponentChildren;
@ -9,7 +9,7 @@ export type Props = {
url: URL; url: URL;
description?: string; description?: string;
context?: { type: string }; context?: { type: string };
searchResults?: SearchResult; searchResults?: GenericResource[];
}; };
export const MainLayout = ( export const MainLayout = (

View File

@ -33,6 +33,7 @@
"@std/yaml": "jsr:@std/yaml@^1.0.5", "@std/yaml": "jsr:@std/yaml@^1.0.5",
"drizzle-kit": "npm:drizzle-kit@^0.30.1", "drizzle-kit": "npm:drizzle-kit@^0.30.1",
"drizzle-orm": "npm:drizzle-orm@^0.38.3", "drizzle-orm": "npm:drizzle-orm@^0.38.3",
"fuzzysort": "npm:fuzzysort@^3.1.0",
"preact": "https://esm.sh/preact@10.22.0", "preact": "https://esm.sh/preact@10.22.0",
"preact-render-to-string": "https://esm.sh/*preact-render-to-string@6.2.2", "preact-render-to-string": "https://esm.sh/*preact-render-to-string@6.2.2",
"preact/": "https://esm.sh/preact@10.22.0/", "preact/": "https://esm.sh/preact@10.22.0/",
@ -42,7 +43,6 @@
"tailwindcss/plugin": "npm:/tailwindcss@^3.4.17/plugin.js", "tailwindcss/plugin": "npm:/tailwindcss@^3.4.17/plugin.js",
"camelcase-css": "npm:camelcase-css", "camelcase-css": "npm:camelcase-css",
"tsx": "npm:tsx@^4.19.2", "tsx": "npm:tsx@^4.19.2",
"typesense": "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/mod.ts",
"yaml": "https://deno.land/std@0.197.0/yaml/mod.ts", "yaml": "https://deno.land/std@0.197.0/yaml/mod.ts",
"zod": "https://deno.land/x/zod@v3.21.4/mod.ts", "zod": "https://deno.land/x/zod@v3.21.4/mod.ts",
"fs": "https://deno.land/std/fs/mod.ts", "fs": "https://deno.land/std/fs/mod.ts",

View File

@ -8,16 +8,6 @@ services:
volumes: volumes:
- ./data/redis-data:/data - ./data/redis-data:/data
typesense:
image: typesense/typesense:0.24.1
restart: on-failure
ports:
- "8108:8108"
volumes:
- ./data/typesense-data:/data
env_file: .env
command: '--data-dir /data'
volumes: volumes:
redis-data: redis-data:
typesense-data: typesense-data:

View File

@ -21,14 +21,12 @@ import * as $api_movies_name_ from "./routes/api/movies/[name].ts";
import * as $api_movies_enhance_name_ from "./routes/api/movies/enhance/[name].ts"; import * as $api_movies_enhance_name_ from "./routes/api/movies/enhance/[name].ts";
import * as $api_movies_index from "./routes/api/movies/index.ts"; import * as $api_movies_index from "./routes/api/movies/index.ts";
import * as $api_query_index from "./routes/api/query/index.ts"; import * as $api_query_index from "./routes/api/query/index.ts";
import * as $api_query_sync from "./routes/api/query/sync.ts";
import * as $api_recipes_name_ from "./routes/api/recipes/[name].ts"; import * as $api_recipes_name_ from "./routes/api/recipes/[name].ts";
import * as $api_recipes_index from "./routes/api/recipes/index.ts"; import * as $api_recipes_index from "./routes/api/recipes/index.ts";
import * as $api_recommendation_all from "./routes/api/recommendation/all.ts"; import * as $api_recommendation_all from "./routes/api/recommendation/all.ts";
import * as $api_recommendation_data from "./routes/api/recommendation/data.ts"; import * as $api_recommendation_data from "./routes/api/recommendation/data.ts";
import * as $api_recommendation_index from "./routes/api/recommendation/index.ts"; import * as $api_recommendation_index from "./routes/api/recommendation/index.ts";
import * as $api_recommendation_movie_id_ from "./routes/api/recommendation/movie/[id].ts"; import * as $api_recommendation_movie_id_ from "./routes/api/recommendation/movie/[id].ts";
import * as $api_resources from "./routes/api/resources.ts";
import * as $api_series_name_ from "./routes/api/series/[name].ts"; import * as $api_series_name_ from "./routes/api/series/[name].ts";
import * as $api_series_enhance_name_ from "./routes/api/series/enhance/[name].ts"; import * as $api_series_enhance_name_ from "./routes/api/series/enhance/[name].ts";
import * as $api_series_index from "./routes/api/series/index.ts"; import * as $api_series_index from "./routes/api/series/index.ts";
@ -80,14 +78,12 @@ const manifest = {
"./routes/api/movies/enhance/[name].ts": $api_movies_enhance_name_, "./routes/api/movies/enhance/[name].ts": $api_movies_enhance_name_,
"./routes/api/movies/index.ts": $api_movies_index, "./routes/api/movies/index.ts": $api_movies_index,
"./routes/api/query/index.ts": $api_query_index, "./routes/api/query/index.ts": $api_query_index,
"./routes/api/query/sync.ts": $api_query_sync,
"./routes/api/recipes/[name].ts": $api_recipes_name_, "./routes/api/recipes/[name].ts": $api_recipes_name_,
"./routes/api/recipes/index.ts": $api_recipes_index, "./routes/api/recipes/index.ts": $api_recipes_index,
"./routes/api/recommendation/all.ts": $api_recommendation_all, "./routes/api/recommendation/all.ts": $api_recommendation_all,
"./routes/api/recommendation/data.ts": $api_recommendation_data, "./routes/api/recommendation/data.ts": $api_recommendation_data,
"./routes/api/recommendation/index.ts": $api_recommendation_index, "./routes/api/recommendation/index.ts": $api_recommendation_index,
"./routes/api/recommendation/movie/[id].ts": $api_recommendation_movie_id_, "./routes/api/recommendation/movie/[id].ts": $api_recommendation_movie_id_,
"./routes/api/resources.ts": $api_resources,
"./routes/api/series/[name].ts": $api_series_name_, "./routes/api/series/[name].ts": $api_series_name_,
"./routes/api/series/enhance/[name].ts": $api_series_enhance_name_, "./routes/api/series/enhance/[name].ts": $api_series_enhance_name_,
"./routes/api/series/index.ts": $api_series_index, "./routes/api/series/index.ts": $api_series_index,

View File

@ -1,5 +1,5 @@
import { Signal, useSignal } from "@preact/signals"; import { Signal, useSignal } from "@preact/signals";
import { useEffect, useRef } from "preact/hooks"; import { useRef } from "preact/hooks";
import { useEventListener } from "@lib/hooks/useEventListener.ts"; import { useEventListener } from "@lib/hooks/useEventListener.ts";
import { menus } from "@islands/KMenu/commands.ts"; import { menus } from "@islands/KMenu/commands.ts";
import { MenuEntry } from "@islands/KMenu/types.ts"; import { MenuEntry } from "@islands/KMenu/types.ts";

View File

@ -31,13 +31,13 @@ export const menus: Record<string, Menu> = {
title: "Login", title: "Login",
icon: "IconLogin", icon: "IconLogin",
cb: () => { cb: () => {
const url = new URL(window.location.href); const url = new URL(globalThis.location.href);
url.pathname = "/api/auth/login"; url.pathname = "/api/auth/login";
url.searchParams.set( url.searchParams.set(
"redirect", "redirect",
encodeURIComponent(window.location.pathname), encodeURIComponent(globalThis.location.pathname),
); );
window.location.href = url.href; globalThis.location.href = url.href;
}, },
visible: () => { visible: () => {
return !getCookie("session_cookie"); return !getCookie("session_cookie");
@ -47,35 +47,24 @@ export const menus: Record<string, Menu> = {
title: "Search", title: "Search",
icon: "IconSearch", icon: "IconSearch",
cb: () => { cb: () => {
window.location.href += "?q="; globalThis.location.href += "?q=";
}, },
visible: () => { visible: () => {
return !!getCookie("session_cookie") && window.location.search === ""; return !!getCookie("session_cookie") &&
globalThis.location.search === "";
}, },
}, },
{ {
title: "Logout", title: "Logout",
icon: "IconLogout", icon: "IconLogout",
cb: () => { cb: () => {
const url = new URL(window.location.href); const url = new URL(globalThis.location.href);
url.pathname = "/api/auth/logout"; url.pathname = "/api/auth/logout";
url.searchParams.set( url.searchParams.set(
"redirect", "redirect",
encodeURIComponent(window.location.pathname), encodeURIComponent(globalThis.location.pathname),
); );
window.location.href = url.href; globalThis.location.href = url.href;
},
visible: () => {
return !!getCookie("session_cookie");
},
},
{
title: "Sync Typesense",
icon: "IconStarFilled",
cb: () => {
fetch("/api/query/sync", {
method: "POST",
});
}, },
visible: () => { visible: () => {
return !!getCookie("session_cookie"); return !!getCookie("session_cookie");

View File

@ -2,7 +2,7 @@ import { useEffect, useRef } from "preact/hooks";
import useDebouncedCallback from "@lib/hooks/useDebouncedCallback.ts"; import useDebouncedCallback from "@lib/hooks/useDebouncedCallback.ts";
import { IconLoader2, IconSearch } from "@components/icons.tsx"; import { IconLoader2, IconSearch } from "@components/icons.tsx";
import { useEventListener } from "@lib/hooks/useEventListener.ts"; import { useEventListener } from "@lib/hooks/useEventListener.ts";
import { SearchResult } from "@lib/types.ts"; import { GenericResource } from "@lib/types.ts";
import { resources } from "@lib/resources.ts"; import { resources } from "@lib/resources.ts";
import { getCookie } from "@lib/string.ts"; import { getCookie } from "@lib/string.ts";
import { IS_BROWSER } from "$fresh/runtime.ts"; import { IS_BROWSER } from "$fresh/runtime.ts";
@ -17,13 +17,13 @@ export async function fetchQueryResource(url: URL, type = "") {
const status = url.searchParams.get("status"); const status = url.searchParams.get("status");
try { try {
url.pathname = "/api/resources"; url.pathname = "/api/query";
url.searchParams.set("q", encodeURIComponent(query || "*")); url.searchParams.set("q", encodeURIComponent(query || ""));
if (status) { if (status) {
url.searchParams.set("status", "not-seen"); url.searchParams.set("status", "not-seen");
} }
if (type) { if (type) {
url.searchParams.set("type", type); url.searchParams.set("types", type);
} }
const response = await fetch(url); const response = await fetch(url);
const jsonData = await response.json(); const jsonData = await response.json();
@ -39,9 +39,9 @@ export const RedirectSearchHandler = () => {
if (e?.target?.nodeName == "INPUT") return; if (e?.target?.nodeName == "INPUT") return;
if ( if (
e.key === "?" && e.key === "?" &&
window.location.search === "" globalThis.location.search === ""
) { ) {
window.location.href += "?q=*"; globalThis.location.href += "?q=";
} }
}, IS_BROWSER ? document?.body : undefined); }, IS_BROWSER ? document?.body : undefined);
} }
@ -50,12 +50,12 @@ export const RedirectSearchHandler = () => {
}; };
const SearchResultImage = ({ src }: { src: string }) => { const SearchResultImage = ({ src }: { src: string }) => {
const imageSrc = `/api/images?image=${src}&width=50&height=50`;
return ( return (
<Image <Image
class="object-cover w-12 h-12 rounded-full" class="object-cover w-12 h-12 rounded-full"
src={imageSrc} width="50"
height="50"
src={src}
alt="preview image" alt="preview image"
/> />
); );
@ -63,13 +63,12 @@ const SearchResultImage = ({ src }: { src: string }) => {
export const SearchResultItem = ( export const SearchResultItem = (
{ item, showEmoji = false }: { { item, showEmoji = false }: {
item: NonNullable<SearchResult["hits"]>[number]; item: GenericResource;
showEmoji?: boolean; showEmoji?: boolean;
}, },
) => { ) => {
const doc = item.document; const resourceType = resources[item.type];
const resourceType = resources[doc.type]; const href = resourceType ? `${resourceType.link}/${item.id}` : "";
const href = resourceType ? `${resourceType.link}/${doc.id}` : "";
return ( return (
<a <a
href={href} href={href}
@ -78,21 +77,21 @@ export const SearchResultItem = (
{showEmoji && resourceType {showEmoji && resourceType
? <Emoji class="w-7 h-7" name={resourceType.emoji} /> ? <Emoji class="w-7 h-7" name={resourceType.emoji} />
: ""} : ""}
{doc?.image && <SearchResultImage src={doc.image} />} {item.meta?.image && <SearchResultImage src={item.meta?.image} />}
{doc?.name} {item?.name}
</a> </a>
); );
}; };
export const SearchResultList = ( export const SearchResultList = (
{ result, showEmoji }: { result: SearchResult; showEmoji?: boolean }, { result, showEmoji }: { result: GenericResource[]; showEmoji?: boolean },
) => { ) => {
return ( return (
<div class="mt-4"> <div class="mt-4">
{result?.hits {result?.length
? ( ? (
<div class="flex flex-col gap-4"> <div class="flex flex-col gap-4">
{result.hits.map((hit) => ( {result.map((hit) => (
<SearchResultItem item={hit} showEmoji={showEmoji} /> <SearchResultItem item={hit} showEmoji={showEmoji} />
))} ))}
</div> </div>
@ -106,17 +105,17 @@ const Search = (
{ q = "*", type, results }: { { q = "*", type, results }: {
q: string; q: string;
type?: string; type?: string;
results?: SearchResult; results?: GenericResource[];
}, },
) => { ) => {
const searchQuery = useSignal(q); const searchQuery = useSignal(q);
const data = useSignal<SearchResult | undefined>(results); const data = useSignal<GenericResource[] | undefined>(results);
const isLoading = useSignal(false); const isLoading = useSignal(false);
const showSeenStatus = useSignal(false); const showSeenStatus = useSignal(false);
const inputRef = useRef<HTMLInputElement>(null); const inputRef = useRef<HTMLInputElement>(null);
if ("history" in globalThis) { if ("history" in globalThis) {
const u = new URL(window.location.href); const u = new URL(globalThis.location.href);
if (u.searchParams.get("q") !== searchQuery.value) { if (u.searchParams.get("q") !== searchQuery.value) {
u.searchParams.set("q", searchQuery.value); u.searchParams.set("q", searchQuery.value);
} }
@ -126,14 +125,14 @@ const Search = (
u.searchParams.delete("rating"); u.searchParams.delete("rating");
} }
window.history.replaceState({}, "", u); globalThis.history.replaceState({}, "", u);
} }
const fetchData = async () => { const fetchData = async () => {
try { try {
isLoading.value = true; isLoading.value = true;
const jsonData = await fetchQueryResource( const jsonData = await fetchQueryResource(
new URL(window?.location.href), new URL(globalThis?.location.href),
type, type,
); );
data.value = jsonData; data.value = jsonData;
@ -188,7 +187,7 @@ const Search = (
<Checkbox label="seen" checked={showSeenStatus} /> <Checkbox label="seen" checked={showSeenStatus} />
<Rating rating={4} /> <Rating rating={4} />
</header> </header>
{data?.value?.hits?.length && !isLoading.value {data.value?.length && !isLoading.value
? <SearchResultList showEmoji={!type} result={data.value} /> ? <SearchResultList showEmoji={!type} result={data.value} />
: isLoading.value : isLoading.value
? <div /> ? <div />

View File

@ -82,7 +82,7 @@ export function createCrud<T extends GenericResource>(
return addThumbnailToResource(res); return addThumbnailToResource(res);
} }
return res; return { ...res, content };
} }
function create(id: string, content: string | ArrayBuffer | T) { function create(id: string, content: string | ArrayBuffer | T) {
const path = pathFromId(id); const path = pathFromId(id);

View File

@ -11,7 +11,6 @@ import remarkFrontmatter, {
import { SILVERBULLET_SERVER } from "@lib/env.ts"; import { SILVERBULLET_SERVER } from "@lib/env.ts";
import { fixRenderedMarkdown } from "@lib/helpers.ts"; import { fixRenderedMarkdown } from "@lib/helpers.ts";
import { createLogger } from "@lib/log.ts"; import { createLogger } from "@lib/log.ts";
import * as typesense from "@lib/typesense.ts";
import { db } from "@lib/sqlite/sqlite.ts"; import { db } from "@lib/sqlite/sqlite.ts";
import { documentTable } from "@lib/sqlite/schema.ts"; import { documentTable } from "@lib/sqlite/schema.ts";
import { eq } from "drizzle-orm/sql"; import { eq } from "drizzle-orm/sql";
@ -59,8 +58,6 @@ export function createDocument(
log.info("creating document", { name }); log.info("creating document", { name });
typesense.synchronize();
return fetch(SILVERBULLET_SERVER + "/" + name, { return fetch(SILVERBULLET_SERVER + "/" + name, {
body: content, body: content,
method: "PUT", method: "PUT",

View File

@ -22,13 +22,9 @@ export const SESSION_DURATION = duration ? +duration : (60 * 60 * 24);
export const JWT_SECRET = Deno.env.get("JWT_SECRET"); export const JWT_SECRET = Deno.env.get("JWT_SECRET");
export const TYPESENSE_URL = Deno.env.get("TYPESENSE_URL") ||
"http://localhost:8108";
export const TYPESENSE_API_KEY = Deno.env.get("TYPESENSE_API_KEY");
export const DATA_DIR = Deno.env.has("DATA_DIR") export const DATA_DIR = Deno.env.has("DATA_DIR")
? path.resolve(Deno.env.get("DATA_DIR")!) ? path.resolve(Deno.env.get("DATA_DIR")!)
: path.resolve(Deno.cwd(), "data"); : path.resolve(Deno.cwd(), "data");
export const LOG_LEVEL: string = Deno.env.get("LOG_LEVEL") || export const LOG_LEVEL: string = Deno.env.get("LOG_LEVEL") ||
"debug"; "warn";

View File

@ -1,17 +1,20 @@
import { resources } from "@lib/resources.ts"; import { resources } from "@lib/resources.ts";
import { SearchResult } from "@lib/types.ts"; import fuzzysort from "npm:fuzzysort";
import { getTypeSenseClient } from "@lib/typesense.ts"; import { GenericResource } from "@lib/types.ts";
import { extractHashTags } from "@lib/string.ts"; import { extractHashTags } from "@lib/string.ts";
import { getAllMovies, Movie } from "@lib/resource/movies.ts";
import { Article, getAllArticles } from "@lib/resource/articles.ts";
import { getAllRecipes, Recipe } from "@lib/resource/recipes.ts";
import { getAllSeries, Series } from "@lib/resource/series.ts";
type ResourceType = keyof typeof resources; type ResourceType = keyof typeof resources;
type SearchParams = { type SearchParams = {
q: string; q: string;
type?: ResourceType; types?: string[];
tags?: string[]; tags?: string[];
rating?: string; rating?: number;
author?: string; authors?: string[];
query_by?: string;
}; };
export function parseResourceUrl(_url: string | URL): SearchParams | undefined { export function parseResourceUrl(_url: string | URL): SearchParams | undefined {
@ -31,56 +34,60 @@ export function parseResourceUrl(_url: string | URL): SearchParams | undefined {
return { return {
q: query, q: query,
type: url.searchParams.get("type") as ResourceType || undefined, types: url.searchParams.get("type")?.split(",") as ResourceType[] ||
undefined,
tags: hashTags, tags: hashTags,
rating: url.searchParams.get("rating") || undefined, rating: url.searchParams.has("rating")
query_by: url.searchParams.get("query_by") || undefined, ? parseInt(url.searchParams.get("rating")!)
: undefined,
}; };
} catch (_err) { } catch (_err) {
return undefined; return undefined;
} }
} }
const isResource = (
item: Movie | Series | Article | Recipe | boolean,
): item is Movie | Series | Article | Recipe => {
return !!item;
};
export async function searchResource( export async function searchResource(
{ q, query_by = "name,description,author,tags", tags = [], type, rating }: { q, tags = [], types, authors, rating }: SearchParams,
SearchParams, ): Promise<GenericResource[]> {
): Promise<SearchResult> { console.log("searchResource", { q, tags, types, authors, rating });
const typesenseClient = await getTypeSenseClient();
if (!typesenseClient) {
throw new Error("Query not available");
}
const filter_by: string[] = []; let resources = (await Promise.all([
(!types || types.includes("movie")) && getAllMovies(),
if (type) { (!types || types.includes("series")) && getAllSeries(),
filter_by.push(`type:=${type}`); (!types || types.includes("article")) && getAllArticles(),
} (!types || types.includes("recipe")) && getAllRecipes(),
])).flat().filter(isResource);
if (tags?.length) { if (tags?.length) {
filter_by.push(`tags:[${tags.map((t) => `\`${t}\``).join(",")}]`); resources = resources.filter((r) => {
for (const tag of tags) { return tags?.every((t) => r.tags.includes(t));
q = q.replaceAll(`#${tag}`, "");
}
if (!q.trim().length) {
q = "*";
}
}
if (typeof rating !== "undefined") {
if (rating === "null") {
filter_by.push(`rating: null`);
} else {
filter_by.push(`rating: ${rating}`);
}
}
return await typesenseClient.collections("resources")
.documents().search({
q,
query_by,
facet_by: "rating,author,tags",
max_facet_values: 10,
filter_by: filter_by.join(" && "),
per_page: 50,
}); });
}
if (authors?.length) {
resources = resources.filter((r) => {
return r?.meta?.author && authors.includes(r?.meta?.author);
});
}
if (rating) {
resources = resources.filter((r) => {
return r?.meta?.rating && r.meta.rating >= rating;
});
}
if (q.length && q !== "*") {
const results = fuzzysort.go(q, resources, {
keys: ["content", "name", "description"],
threshold: 0.3,
});
resources = results.map((r) => r.obj);
}
return resources;
} }

View File

@ -1,4 +1,3 @@
import { SearchResponse } from "https://raw.githubusercontent.com/bradenmacdonald/typesense-deno/main/src/Typesense/Documents.ts";
import { resources } from "@lib/resources.ts"; import { resources } from "@lib/resources.ts";
export interface TMDBMovie { export interface TMDBMovie {
@ -39,6 +38,7 @@ export type GenericResource = {
id: string; id: string;
tags?: string[]; tags?: string[];
type: keyof typeof resources; type: keyof typeof resources;
content?: string;
meta?: { meta?: {
image?: string; image?: string;
author?: string; author?: string;
@ -58,7 +58,7 @@ export interface GiteaOauthUser {
groups: any; groups: any;
} }
export type TypesenseDocument = { export type SearchResult = {
id: string; id: string;
name: string; name: string;
type: keyof typeof resources; type: keyof typeof resources;
@ -68,5 +68,3 @@ export type TypesenseDocument = {
description?: string; description?: string;
image?: string; image?: string;
}; };
export type SearchResult = SearchResponse<TypesenseDocument>;

View File

@ -1,178 +0,0 @@
import { Client } from "typesense";
import { TYPESENSE_API_KEY, TYPESENSE_URL } from "@lib/env.ts";
import { getAllMovies } from "@lib/resource/movies.ts";
import { getAllRecipes } from "@lib/resource/recipes.ts";
import { getAllArticles } from "@lib/resource/articles.ts";
import { createLogger } from "@lib/log.ts";
import { debounce } from "https://deno.land/std@0.193.0/async/mod.ts";
import { getAllSeries } from "@lib/resource/series.ts";
import { TypesenseDocument } from "@lib/types.ts";
const log = createLogger("typesense");
function sanitizeStringForTypesense(input: string) {
// Remove backslashes
const withoutBackslashes = input.replace(/\\/g, "");
// Remove control characters other than carriage return and line feed
const withoutControlCharacters = withoutBackslashes.replace(
/[\x00-\x09\x0B\x0C\x0E-\x1F\x7F]/g,
"",
);
// Remove Unicode characters above U+FFFF
const withoutUnicodeAboveFFFF = withoutControlCharacters.replace(
/[\uD800-\uDBFF][\uDC00-\uDFFF]/g,
"",
);
return withoutUnicodeAboveFFFF;
}
// Use a promise to initialize the client as needed, rather than at import time.
let clientPromise: Promise<Client | null> | undefined;
export function getTypeSenseClient(): Promise<Client | null> {
if (clientPromise === undefined) {
let typesenseUrl: URL;
try {
typesenseUrl = new URL(TYPESENSE_URL);
} catch (_err) {
return Promise.resolve(null);
}
clientPromise = new Promise((resolve) => {
if (!TYPESENSE_API_KEY) {
return resolve(null);
}
const client = new Client({
nodes: [{
host: typesenseUrl.hostname,
port: +typesenseUrl.port || 8108,
protocol: typesenseUrl.protocol.slice(0, -1),
}],
apiKey: TYPESENSE_API_KEY,
connectionTimeoutSeconds: 2,
});
resolve(client);
});
}
return clientPromise;
}
async function initializeTypesense() {
try {
const client = await getTypeSenseClient();
if (!client) return;
// Create the "resources" collection if it doesn't exist
const collections = await client.collections().retrieve();
const resourcesCollection = collections.find((collection) =>
collection.name === "resources"
);
if (!resourcesCollection) {
await client.collections().create({
name: "resources",
fields: [
{ name: "name", type: "string" },
{ name: "type", type: "string", facet: true },
{ name: "date", type: "string", optional: true },
{ name: "author", type: "string", facet: true, optional: true },
{ name: "rating", type: "int32", facet: true },
{ name: "tags", type: "string[]", facet: true },
{ name: "description", type: "string", optional: true },
{ name: "image", type: "string", optional: true },
],
default_sorting_field: "rating", // Default field for sorting
});
log.info('created "resources" collection');
} else {
log.info('collection "resources" already exists.');
}
} catch (error) {
log.error("error initializing", error);
}
}
const init = initializeTypesense();
export async function createTypesenseDocument(doc: TypesenseDocument) {
const client = await getTypeSenseClient();
if (!client) return;
// await client.collections("resources").documents().create(
// doc,
// { action: "upsert" },
// );
}
async function synchronizeWithTypesense() {
return;
await init;
try {
const allResources = (await Promise.all([
getAllMovies(),
getAllArticles(),
getAllRecipes(),
getAllSeries(),
])).flat(); // Replace with your function to get all resources from the database
const client = await getTypeSenseClient();
if (!client) return;
// Convert the list of documents to Typesense compatible format (array of objects)
const typesenseDocuments = allResources.map((resource) => {
return {
id: resource.id, // Convert the document ID to a string, as Typesense only supports string IDs
name: sanitizeStringForTypesense(resource.name),
description: sanitizeStringForTypesense(
resource?.description || resource?.content || "",
),
author: resource.meta?.author,
image: resource.meta?.image,
tags: resource?.tags || [],
rating: resource.meta?.rating || 0,
date: resource.meta?.date?.toString() || "",
type: resource.type,
};
});
return;
await client.collections("resources").documents().import(
typesenseDocuments,
{ action: "upsert" },
);
// Get all the IDs of documents currently indexed in Typesense
const allTypesenseDocuments = await client.collections("resources")
.documents().search({
q: "*",
query_by: "name,type,date,description",
per_page: 250,
limit_hits: 9999,
});
const deletedDocumentIds = allTypesenseDocuments.hits
?.map((doc) => doc?.document?.id)
?.filter((id) =>
// Find deleted document IDs by comparing the Typesense document IDs with the current list of resources
!allResources.some((resource) => resource.id.toString() === id)
).map((id) => client.collections("resources").documents(id).delete());
// Delete the documents with IDs found in deletedDocumentIds
if (deletedDocumentIds) {
await Promise.all(
deletedDocumentIds,
);
}
log.info("data synchronized");
} catch (error) {
log.error("error synchronizing", error);
}
}
// Call the synchronizeWithTypesense function to trigger the synchronization
synchronizeWithTypesense();
export const synchronize = debounce(synchronizeWithTypesense, 1000 * 60 * 5);

View File

@ -1,15 +1,7 @@
import { Handlers } from "$fresh/server.ts"; import { Handlers } from "$fresh/server.ts";
import { json } from "@lib/helpers.ts"; import { json } from "@lib/helpers.ts";
import { getAllMovies, Movie } from "@lib/resource/movies.ts";
import { Article, getAllArticles } from "@lib/resource/articles.ts";
import { getAllRecipes, Recipe } from "@lib/resource/recipes.ts";
import { AccessDeniedError } from "@lib/errors.ts"; import { AccessDeniedError } from "@lib/errors.ts";
import { searchResource } from "@lib/search.ts";
const isResource = (
item: Movie | Article | Recipe | boolean,
): item is Movie | Article | Recipe => {
return !!item;
};
export const handler: Handlers = { export const handler: Handlers = {
async GET(req, ctx) { async GET(req, ctx) {
@ -20,27 +12,16 @@ export const handler: Handlers = {
const url = new URL(req.url); const url = new URL(req.url);
const types = url.searchParams.get("type")?.split(", "); const types = url.searchParams.get("types")?.split(",");
let resources = (await Promise.all([
(!types || types.includes("movie")) && getAllMovies(),
(!types || types.includes("article")) && getAllArticles(),
(!types || types.includes("recipe")) && getAllRecipes(),
])).flat().filter(isResource);
const tags = url.searchParams?.get("tags")?.split(","); const tags = url.searchParams?.get("tags")?.split(",");
if (tags?.length) { const authors = url.searchParams?.get("authors")?.split(",");
resources = resources.filter((r) => {
return tags?.every((t) => r.tags.includes(t));
});
}
const authors = url.searchParams?.get("author")?.split(","); const resources = await searchResource({
if (authors?.length) { q: url.searchParams.get("q") || "",
resources = resources.filter((r) => { types,
return r?.meta?.author && authors.includes(r?.meta?.author); tags,
}); authors,
} });
return json(resources); return json(resources);
}, },

View File

@ -1,16 +0,0 @@
import { AccessDeniedError } from "@lib/errors.ts";
import { Handlers } from "$fresh/server.ts";
import { synchronize } from "@lib/typesense.ts";
export const handler: Handlers = {
POST(_, ctx) {
const session = ctx.state.session;
if (!session) {
throw new AccessDeniedError();
}
synchronize();
return new Response("OK");
},
};

View File

@ -1,21 +0,0 @@
import { Handlers } from "$fresh/server.ts";
import { AccessDeniedError, BadRequestError } from "@lib/errors.ts";
import { getTypeSenseClient } from "@lib/typesense.ts";
import { json } from "@lib/helpers.ts";
import { parseResourceUrl, searchResource } from "@lib/search.ts";
export const handler: Handlers = {
async GET(req, ctx) {
const session = ctx.state.session;
if (!session) {
throw new AccessDeniedError();
}
const searchParams = parseResourceUrl(req.url);
// Perform the Typesense search
const searchResults = await searchResource(searchParams);
return json(searchResults);
},
};

View File

@ -1,29 +1,30 @@
import { Handlers, PageProps } from "$fresh/server.ts"; import { Handlers, PageProps } from "$fresh/server.ts";
import { MainLayout } from "@components/layouts/main.tsx"; import { MainLayout } from "@components/layouts/main.tsx";
import { Article, getAllArticles } from "@lib/resource/articles.ts"; import { Article, getAllArticles } from "@lib/resource/articles.ts";
import { Card } from "@components/Card.tsx";
import { KMenu } from "@islands/KMenu.tsx"; import { KMenu } from "@islands/KMenu.tsx";
import { Grid } from "@components/Grid.tsx"; import { Grid } from "@components/Grid.tsx";
import { IconArrowLeft } from "@components/icons.tsx"; import { IconArrowLeft } from "@components/icons.tsx";
import { RedirectSearchHandler } from "@islands/Search.tsx"; import { RedirectSearchHandler } from "@islands/Search.tsx";
import { parseResourceUrl, searchResource } from "@lib/search.ts"; import { parseResourceUrl, searchResource } from "@lib/search.ts";
import { SearchResult } from "@lib/types.ts"; import { GenericResource } from "@lib/types.ts";
import { ResourceCard } from "@components/Card.tsx"; import { ResourceCard } from "@components/Card.tsx";
export const handler: Handlers< export const handler: Handlers<
{ articles: Article[] | null; searchResults?: SearchResult } { articles: Article[] | null; searchResults?: GenericResource[] }
> = { > = {
async GET(req, ctx) { async GET(req, ctx) {
const articles = await getAllArticles(); const articles = await getAllArticles();
const searchParams = parseResourceUrl(req.url); const searchParams = parseResourceUrl(req.url);
const searchResults = searchParams && const searchResults = searchParams &&
await searchResource({ ...searchParams, type: "article" }); await searchResource({ ...searchParams, types: ["article"] });
return ctx.render({ articles, searchResults }); return ctx.render({ articles, searchResults });
}, },
}; };
export default function Greet( export default function Greet(
props: PageProps<{ articles: Article[] | null; searchResults: SearchResult }>, props: PageProps<
{ articles: Article[] | null; searchResults: GenericResource[] }
>,
) { ) {
const { articles, searchResults } = props.data; const { articles, searchResults } = props.data;
return ( return (

View File

@ -6,16 +6,18 @@ import { IconArrowLeft } from "@components/icons.tsx";
import { KMenu } from "@islands/KMenu.tsx"; import { KMenu } from "@islands/KMenu.tsx";
import { RedirectSearchHandler } from "@islands/Search.tsx"; import { RedirectSearchHandler } from "@islands/Search.tsx";
import { parseResourceUrl, searchResource } from "@lib/search.ts"; import { parseResourceUrl, searchResource } from "@lib/search.ts";
import { SearchResult } from "@lib/types.ts"; import { GenericResource } from "@lib/types.ts";
import { PageProps } from "$fresh/server.ts"; import { PageProps } from "$fresh/server.ts";
export default async function Greet( export default async function Greet(
props: PageProps<{ movies: Movie[] | null; searchResults: SearchResult }>, props: PageProps<
{ movies: Movie[] | null; searchResults: GenericResource[] }
>,
) { ) {
const allMovies = await getAllMovies(); const allMovies = await getAllMovies();
const searchParams = parseResourceUrl(props.url); const searchParams = parseResourceUrl(props.url);
const searchResults = searchParams && const searchResults = searchParams &&
await searchResource({ ...searchParams, type: "movie" }); await searchResource({ ...searchParams, types: ["movie"] });
const movies = allMovies.sort((a, b) => const movies = allMovies.sort((a, b) =>
a?.meta?.rating > b?.meta?.rating ? -1 : 1 a?.meta?.rating > b?.meta?.rating ? -1 : 1
); );

View File

@ -6,23 +6,25 @@ import { IconArrowLeft } from "@components/icons.tsx";
import { KMenu } from "@islands/KMenu.tsx"; import { KMenu } from "@islands/KMenu.tsx";
import { RedirectSearchHandler } from "@islands/Search.tsx"; import { RedirectSearchHandler } from "@islands/Search.tsx";
import { parseResourceUrl, searchResource } from "@lib/search.ts"; import { parseResourceUrl, searchResource } from "@lib/search.ts";
import { SearchResult } from "@lib/types.ts"; import { GenericResource } from "@lib/types.ts";
import { ResourceCard } from "@components/Card.tsx"; import { ResourceCard } from "@components/Card.tsx";
export const handler: Handlers< export const handler: Handlers<
{ recipes: Recipe[] | null; searchResults?: SearchResult } { recipes: Recipe[] | null; searchResults?: GenericResource[] }
> = { > = {
async GET(req, ctx) { async GET(req, ctx) {
const recipes = await getAllRecipes(); const recipes = await getAllRecipes();
const searchParams = parseResourceUrl(req.url); const searchParams = parseResourceUrl(req.url);
const searchResults = searchParams && const searchResults = searchParams &&
await searchResource({ ...searchParams, type: "recipe" }); await searchResource({ ...searchParams, types: ["recipe"] });
return ctx.render({ recipes, searchResults }); return ctx.render({ recipes, searchResults });
}, },
}; };
export default function Greet( export default function Greet(
props: PageProps<{ recipes: Recipe[] | null; searchResults: SearchResult }>, props: PageProps<
{ recipes: Recipe[] | null; searchResults: GenericResource[] }
>,
) { ) {
const { recipes, searchResults } = props.data; const { recipes, searchResults } = props.data;
return ( return (

View File

@ -7,22 +7,24 @@ import { RedirectSearchHandler } from "@islands/Search.tsx";
import { KMenu } from "@islands/KMenu.tsx"; import { KMenu } from "@islands/KMenu.tsx";
import { ResourceCard } from "@components/Card.tsx"; import { ResourceCard } from "@components/Card.tsx";
import { parseResourceUrl, searchResource } from "@lib/search.ts"; import { parseResourceUrl, searchResource } from "@lib/search.ts";
import { SearchResult } from "@lib/types.ts"; import { GenericResource } from "@lib/types.ts";
export const handler: Handlers< export const handler: Handlers<
{ series: Series[] | null; searchResults?: SearchResult } { series: Series[] | null; searchResults?: GenericResource[] }
> = { > = {
async GET(req, ctx) { async GET(req, ctx) {
const series = await getAllSeries(); const series = await getAllSeries();
const searchParams = parseResourceUrl(req.url); const searchParams = parseResourceUrl(req.url);
const searchResults = searchParams && const searchResults = searchParams &&
await searchResource({ ...searchParams, type: "series" }); await searchResource({ ...searchParams, types: ["series"] });
return ctx.render({ series, searchResults }); return ctx.render({ series, searchResults });
}, },
}; };
export default function Greet( export default function Greet(
props: PageProps<{ series: Series[] | null; searchResults: SearchResult }>, props: PageProps<
{ series: Series[] | null; searchResults: GenericResource[] }
>,
) { ) {
const { series, searchResults } = props.data; const { series, searchResults } = props.data;