From 4c00d2c82e763ab3e16268dada1152d812d74cf6 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 22:07:32 -0600 Subject: [PATCH] Base AI tag generation on existing tags --- src/photo/actions.ts | 12 +++++++++--- src/photo/ai/index.ts | 30 +++++++++++++++++++++--------- src/photo/ai/server.ts | 14 ++++++++------ src/services/openai.ts | 2 +- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 219c477f..86bc203a 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -9,6 +9,7 @@ import { getPhoto, getPhotos, addTagsToPhotos, + getUniqueTags, } from '@/photo/db/query'; import { GetPhotosOptions, areOptionsSensitive } from './db'; import { @@ -37,7 +38,7 @@ import { blurImageFromUrl, extractImageDataFromBlobPath } from './server'; import { TAG_FAVS, isTagFavs } from '@/tag'; import { convertPhotoToPhotoDbInsert, Photo } from '.'; import { runAuthenticatedAdminServerAction } from '@/auth'; -import { AI_IMAGE_QUERIES, AiImageQuery } from './ai'; +import { AiImageQuery, getAiImageQuery } from './ai'; import { streamOpenAiImageQuery } from '@/services/openai'; import { AI_TEXT_AUTO_GENERATED_FIELDS, @@ -394,8 +395,13 @@ export const streamAiImageQueryAction = async ( imageBase64: string, query: AiImageQuery, ) => - runAuthenticatedAdminServerAction(() => - streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query])); + runAuthenticatedAdminServerAction(async () => { + const existingTags = await getUniqueTags(); + return streamOpenAiImageQuery( + imageBase64, + getAiImageQuery(query, existingTags), + ); + }); export const getImageBlurAction = async (url: string) => runAuthenticatedAdminServerAction(() => blurImageFromUrl(url)); diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index 79287d5b..28ac0045 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -1,5 +1,7 @@ /* eslint-disable max-len */ +import { Tags } from '@/tag'; + export type AiAutoGeneratedField = 'title' | 'caption' | @@ -42,15 +44,25 @@ export type AiImageQuery = 'description-large' | 'description-semantic'; -export const AI_IMAGE_QUERIES: Record = { - 'title': 'Write a compelling title for this image in 3 words or less', - 'caption': 'Write a pithy caption for this image in 6 words or less and no punctuation', - 'title-and-caption': 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"', - 'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs', - 'description-small': 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"', - 'description': 'Describe this image', - 'description-large': 'Describe this image in detail', - 'description-semantic': 'List up to 5 things in this image without description as a comma-separated list', +export const getAiImageQuery = ( + query: AiImageQuery, + existingTags: Tags = [], +): string => { + switch (query) { + case 'title': return 'Write a compelling title for this image in 3 words or less'; + case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation'; + case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"'; + case 'tags': + const tagQuery = 'Describe this image three or less comma-separated keywords with no adjective or adverbs'; + const tags = existingTags.map(({ tag }) => tag).join(', '); + return tags + ? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.` + : tagQuery; + case 'description-small': return 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"'; + case 'description': return 'Describe this image'; + case 'description-large': return 'Describe this image in detail'; + case 'description-semantic': return 'List up to 5 things in this image without description as a comma-separated list'; + } }; export const parseTitleAndCaption = (text: string) => { diff --git a/src/photo/ai/server.ts b/src/photo/ai/server.ts index 163845c1..5355256e 100644 --- a/src/photo/ai/server.ts +++ b/src/photo/ai/server.ts @@ -1,9 +1,10 @@ import { generateOpenAiImageQuery } from '@/services/openai'; import { - AI_IMAGE_QUERIES, AiAutoGeneratedField, + getAiImageQuery, parseTitleAndCaption, } from '.'; +import { getUniqueTags } from '../db/query'; export const generateAiImageQueries = async ( imageBase64?: string, @@ -29,7 +30,7 @@ export const generateAiImageQueries = async ( ) { const titleAndCaption = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['title-and-caption'], + getAiImageQuery('title-and-caption'), ); if (titleAndCaption) { const titleAndCaptionParsed = parseTitleAndCaption(titleAndCaption); @@ -40,28 +41,29 @@ export const generateAiImageQueries = async ( if (textFieldsToGenerate.includes('title')) { title = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['title'], + getAiImageQuery('title'), ); } if (textFieldsToGenerate.includes('caption')) { caption = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['caption'], + getAiImageQuery('caption'), ); } } if (textFieldsToGenerate.includes('tags')) { + const existingTags = await getUniqueTags(); tags = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['tags'], + getAiImageQuery('tags', existingTags), ); } if (textFieldsToGenerate.includes('semantic')) { semanticDescription = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['description-small'], + getAiImageQuery('description-small'), ); } } diff --git a/src/services/openai.ts b/src/services/openai.ts index e4636b8d..72d33a94 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -73,7 +73,7 @@ export const streamOpenAiImageQuery = async ( if (args) { (async () => { - const { textStream } = await streamText(args); + const { textStream } = streamText(args); for await (const delta of textStream) { stream.update(cleanUpAiTextResponse(delta)); }