From 4c00d2c82e763ab3e16268dada1152d812d74cf6 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 22:07:32 -0600 Subject: [PATCH 1/5] Base AI tag generation on existing tags --- src/photo/actions.ts | 12 +++++++++--- src/photo/ai/index.ts | 30 +++++++++++++++++++++--------- src/photo/ai/server.ts | 14 ++++++++------ src/services/openai.ts | 2 +- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 219c477f..86bc203a 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -9,6 +9,7 @@ import { getPhoto, getPhotos, addTagsToPhotos, + getUniqueTags, } from '@/photo/db/query'; import { GetPhotosOptions, areOptionsSensitive } from './db'; import { @@ -37,7 +38,7 @@ import { blurImageFromUrl, extractImageDataFromBlobPath } from './server'; import { TAG_FAVS, isTagFavs } from '@/tag'; import { convertPhotoToPhotoDbInsert, Photo } from '.'; import { runAuthenticatedAdminServerAction } from '@/auth'; -import { AI_IMAGE_QUERIES, AiImageQuery } from './ai'; +import { AiImageQuery, getAiImageQuery } from './ai'; import { streamOpenAiImageQuery } from '@/services/openai'; import { AI_TEXT_AUTO_GENERATED_FIELDS, @@ -394,8 +395,13 @@ export const streamAiImageQueryAction = async ( imageBase64: string, query: AiImageQuery, ) => - runAuthenticatedAdminServerAction(() => - streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query])); + runAuthenticatedAdminServerAction(async () => { + const existingTags = await getUniqueTags(); + return streamOpenAiImageQuery( + imageBase64, + getAiImageQuery(query, existingTags), + ); + }); export const getImageBlurAction = async (url: string) => runAuthenticatedAdminServerAction(() => blurImageFromUrl(url)); diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index 79287d5b..28ac0045 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -1,5 +1,7 @@ /* eslint-disable max-len */ +import { Tags } from '@/tag'; + export type AiAutoGeneratedField = 'title' | 'caption' | @@ -42,15 +44,25 @@ export type AiImageQuery = 'description-large' | 'description-semantic'; -export const AI_IMAGE_QUERIES: Record = { - 'title': 'Write a compelling title for this image in 3 words or less', - 'caption': 'Write a pithy caption for this image in 6 words or less and no punctuation', - 'title-and-caption': 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"', - 'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs', - 'description-small': 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"', - 'description': 'Describe this image', - 'description-large': 'Describe this image in detail', - 'description-semantic': 'List up to 5 things in this image without description as a comma-separated list', +export const getAiImageQuery = ( + query: AiImageQuery, + existingTags: Tags = [], +): string => { + switch (query) { + case 'title': return 'Write a compelling title for this image in 3 words or less'; + case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation'; + case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"'; + case 'tags': + const tagQuery = 'Describe this image three or less comma-separated keywords with no adjective or adverbs'; + const tags = existingTags.map(({ tag }) => tag).join(', '); + return tags + ? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.` + : tagQuery; + case 'description-small': return 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"'; + case 'description': return 'Describe this image'; + case 'description-large': return 'Describe this image in detail'; + case 'description-semantic': return 'List up to 5 things in this image without description as a comma-separated list'; + } }; export const parseTitleAndCaption = (text: string) => { diff --git a/src/photo/ai/server.ts b/src/photo/ai/server.ts index 163845c1..5355256e 100644 --- a/src/photo/ai/server.ts +++ b/src/photo/ai/server.ts @@ -1,9 +1,10 @@ import { generateOpenAiImageQuery } from '@/services/openai'; import { - AI_IMAGE_QUERIES, AiAutoGeneratedField, + getAiImageQuery, parseTitleAndCaption, } from '.'; +import { getUniqueTags } from '../db/query'; export const generateAiImageQueries = async ( imageBase64?: string, @@ -29,7 +30,7 @@ export const generateAiImageQueries = async ( ) { const titleAndCaption = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['title-and-caption'], + getAiImageQuery('title-and-caption'), ); if (titleAndCaption) { const titleAndCaptionParsed = parseTitleAndCaption(titleAndCaption); @@ -40,28 +41,29 @@ export const generateAiImageQueries = async ( if (textFieldsToGenerate.includes('title')) { title = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['title'], + getAiImageQuery('title'), ); } if (textFieldsToGenerate.includes('caption')) { caption = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['caption'], + getAiImageQuery('caption'), ); } } if (textFieldsToGenerate.includes('tags')) { + const existingTags = await getUniqueTags(); tags = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['tags'], + getAiImageQuery('tags', existingTags), ); } if (textFieldsToGenerate.includes('semantic')) { semanticDescription = await generateOpenAiImageQuery( imageBase64, - AI_IMAGE_QUERIES['description-small'], + getAiImageQuery('description-small'), ); } } diff --git a/src/services/openai.ts b/src/services/openai.ts index e4636b8d..72d33a94 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -73,7 +73,7 @@ export const streamOpenAiImageQuery = async ( if (args) { (async () => { - const { textStream } = await streamText(args); + const { textStream } = streamText(args); for await (const delta of textStream) { stream.update(cleanUpAiTextResponse(delta)); } From 2db33fccb0a091ae78c9ffae19fd516a06063176 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 22:13:16 -0600 Subject: [PATCH 2/5] Tweak tooltip underline --- src/photo/PhotoLarge.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/photo/PhotoLarge.tsx b/src/photo/PhotoLarge.tsx index 7f96e1e0..95a85281 100644 --- a/src/photo/PhotoLarge.tsx +++ b/src/photo/PhotoLarge.tsx @@ -265,7 +265,8 @@ export default function PhotoLarge({ {photo.focalLengthIn35MmFormatFormatted} From 440aeeb5610dff6b17f41890733e82ac96362e28 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 22:53:35 -0600 Subject: [PATCH 3/5] Remove 'caption' from default AI text generation --- README.md | 8 ++++---- src/photo/ai/AiButton.tsx | 4 ++-- src/photo/ai/index.ts | 16 +++++++++++----- src/photo/ai/useAiImageQueries.ts | 4 ++-- src/site/SiteChecklistClient.tsx | 4 +++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 93cd1196..38a10462 100644 --- a/README.md +++ b/README.md @@ -77,11 +77,11 @@ _⚠️ READ BEFORE PROCEEDING_ 3. Configure auto-generated fields (optional) - Set which text fields auto-generate when uploading a photo by storing a comma-separated list, e.g., `AI_TEXT_AUTO_GENERATED_FIELDS = title, semantic` - Accepted values: - - `all` (default) - - `title` + - `all` + - `title` (default) - `caption` - - `tags` - - `semantic` + - `tags` (default) + - `semantic` (default) - `none` ### Web Analytics diff --git a/src/photo/ai/AiButton.tsx b/src/photo/ai/AiButton.tsx index 34cef97d..60b5efb6 100644 --- a/src/photo/ai/AiButton.tsx +++ b/src/photo/ai/AiButton.tsx @@ -1,12 +1,12 @@ import { AiContent } from './useAiImageQueries'; import { HiSparkles } from 'react-icons/hi'; -import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.'; +import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.'; import { useMemo } from 'react'; import LoaderButton from '@/components/primitives/LoaderButton'; export default function AiButton({ aiContent, - requestFields = ALL_AI_AUTO_GENERATED_FIELDS, + requestFields = AI_AUTO_GENERATED_FIELDS_ALL, shouldConfirm, className, }: { diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index 28ac0045..3d0ec9f9 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -8,27 +8,33 @@ export type AiAutoGeneratedField = 'tags' | 'semantic' -export const ALL_AI_AUTO_GENERATED_FIELDS: AiAutoGeneratedField[] = [ +export const AI_AUTO_GENERATED_FIELDS_ALL: AiAutoGeneratedField[] = [ 'title', 'caption', 'tags', 'semantic', ]; +export const AI_AUTO_GENERATED_FIELDS_DEFAULT: AiAutoGeneratedField[] = [ + 'title', + 'tags', + 'semantic', +]; + export const parseAiAutoGeneratedFieldsText = ( - text = 'all', + text = AI_AUTO_GENERATED_FIELDS_DEFAULT.join(','), ): AiAutoGeneratedField[] => { const textFormatted = text.trim().toLocaleLowerCase(); if (textFormatted === 'none') { return []; } else if (textFormatted === 'all') { - return ALL_AI_AUTO_GENERATED_FIELDS; + return AI_AUTO_GENERATED_FIELDS_ALL; } else { const fields = textFormatted .toLocaleLowerCase() .split(',') .map(field => field.trim()) - .filter(field => ALL_AI_AUTO_GENERATED_FIELDS + .filter(field => AI_AUTO_GENERATED_FIELDS_ALL .includes(field as AiAutoGeneratedField)); return fields as AiAutoGeneratedField[]; } @@ -53,7 +59,7 @@ export const getAiImageQuery = ( case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation'; case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"'; case 'tags': - const tagQuery = 'Describe this image three or less comma-separated keywords with no adjective or adverbs'; + const tagQuery = 'Describe this image in three or less comma-separated unique keywords, with no adjective or adverbs, that are specific to this image'; const tags = existingTags.map(({ tag }) => tag).join(', '); return tags ? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.` diff --git a/src/photo/ai/useAiImageQueries.ts b/src/photo/ai/useAiImageQueries.ts index 63fb0363..c59711cc 100644 --- a/src/photo/ai/useAiImageQueries.ts +++ b/src/photo/ai/useAiImageQueries.ts @@ -1,7 +1,7 @@ import { useCallback, useEffect, useRef } from 'react'; import useAiImageQuery from './useAiImageQuery'; import useTitleCaptionAiImageQuery from './useTitleCaptionAiImageQuery'; -import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.'; +import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.'; export type AiContent = ReturnType; @@ -59,7 +59,7 @@ export default function useAiImageQueries( const hasRunAllQueriesOnce = useRef(false); const request = useCallback(async ( - fields = ALL_AI_AUTO_GENERATED_FIELDS, + fields = AI_AUTO_GENERATED_FIELDS_ALL, ) => { if (process.env.NODE_ENV !== 'production') { console.log('RUNNING AI QUERIES', fields); diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 1ab614e3..3aa5c4d8 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -424,7 +424,9 @@ export default function SiteChecklistClient({ > Comma-separated fields to auto-generate when uploading photos. Accepted values: title, caption, - tags, description, all, or none (default is {'"all"'}): + tags, description, all, or none + {' '} + (default: {'"title, tags, semantic"'}): {renderEnvVars(['AI_TEXT_AUTO_GENERATED_FIELDS'])} From 3a9e0569c078935bd4db345eb659e911e65711b9 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 23:15:34 -0600 Subject: [PATCH 4/5] Refine AI tag prompt --- src/photo/ai/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index 3d0ec9f9..8c98e9ee 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -59,7 +59,7 @@ export const getAiImageQuery = ( case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation'; case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"'; case 'tags': - const tagQuery = 'Describe this image in three or less comma-separated unique keywords, with no adjective or adverbs, that are specific to this image'; + const tagQuery = 'Describe this image in 1-2 comma-separated unique keywords, with no adjective or adverbs. Avoid using general terms like "nature," "travel," "architecture," or "sky." Use terms that are highly specific to the image and not redundant.'; const tags = existingTags.map(({ tag }) => tag).join(', '); return tags ? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.` From b14b8ca2f4756a5dfca8d0f3245f16a23ab1abdb Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 5 Feb 2025 23:30:34 -0600 Subject: [PATCH 5/5] Fix AI text generation configuration reporting --- __tests__/ai.test.ts | 35 ++++++++++++++++++----------------- src/photo/ai/index.ts | 2 +- src/site/config.ts | 9 ++++++--- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/__tests__/ai.test.ts b/__tests__/ai.test.ts index 22e532c8..d40835ff 100644 --- a/__tests__/ai.test.ts +++ b/__tests__/ai.test.ts @@ -1,68 +1,69 @@ /* eslint-disable quotes */ import { - parseAiAutoGeneratedFieldsText, + AI_AUTO_GENERATED_FIELDS_DEFAULT, + parseAiAutoGeneratedFieldsString, parseTitleAndCaption, } from "@/photo/ai"; describe('AI parses', () => { describe('auto-generated fields', () => { it('with spaces', () => { - expect(parseAiAutoGeneratedFieldsText()) + expect(parseAiAutoGeneratedFieldsString()) + .toStrictEqual(AI_AUTO_GENERATED_FIELDS_DEFAULT); + expect(parseAiAutoGeneratedFieldsString('all')) .toStrictEqual(['title', 'caption', 'tags', 'semantic']); - expect(parseAiAutoGeneratedFieldsText('all')) - .toStrictEqual(['title', 'caption', 'tags', 'semantic']); - expect(parseAiAutoGeneratedFieldsText('title')) + expect(parseAiAutoGeneratedFieldsString('title')) .toStrictEqual(['title']); - expect(parseAiAutoGeneratedFieldsText('title, caption')) + expect(parseAiAutoGeneratedFieldsString('title, caption')) .toStrictEqual(['title', 'caption']); - expect(parseAiAutoGeneratedFieldsText('title, caption, invalid')) + expect(parseAiAutoGeneratedFieldsString('title, caption, invalid')) .toStrictEqual(['title', 'caption']); - expect(parseAiAutoGeneratedFieldsText('title, caption, invalid, tags')) + expect(parseAiAutoGeneratedFieldsString('title, caption, invalid, tags')) .toStrictEqual(['title', 'caption', 'tags']); - expect(parseAiAutoGeneratedFieldsText('none')) + expect(parseAiAutoGeneratedFieldsString('none')) .toStrictEqual([]); }); it('without spaces', () => { - expect(parseAiAutoGeneratedFieldsText('title,caption')) + expect(parseAiAutoGeneratedFieldsString('title,caption')) .toStrictEqual(['title', 'caption']); - expect(parseAiAutoGeneratedFieldsText('title,caption,invalid')) + expect(parseAiAutoGeneratedFieldsString('title,caption,invalid')) .toStrictEqual(['title', 'caption']); - expect(parseAiAutoGeneratedFieldsText('title,caption,invalid,tags')) + expect(parseAiAutoGeneratedFieldsString('title,caption,invalid,tags')) .toStrictEqual(['title', 'caption', 'tags']); }); }); it('received titles and captions', () => { // Complex case expect(parseTitleAndCaption( - `'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'` + `'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'`, )).toStrictEqual({ title: 'Ephemeral Beauty', caption: 'Roses bask in fleeting sunlight', }); // Without surrounding single quotes expect(parseTitleAndCaption( - `Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."` + `Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."`, )).toStrictEqual({ title: 'Ephemeral Beauty', caption: 'Roses bask in fleeting sunlight', }); // Without trailing period expect(parseTitleAndCaption( - `Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight"` + `Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight"`, )).toStrictEqual({ title: 'Ephemeral Beauty', caption: 'Roses bask in fleeting sunlight', }); // Without and quotes expect(parseTitleAndCaption( - `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight` + `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`, )).toStrictEqual({ title: 'Ephemeral Beauty', caption: 'Roses bask in fleeting sunlight', }); // With single space expect(parseTitleAndCaption( - `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight` + `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`, )).toStrictEqual({ title: 'Ephemeral Beauty', caption: 'Roses bask in fleeting sunlight', diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index 8c98e9ee..ad4a1ea5 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -21,7 +21,7 @@ export const AI_AUTO_GENERATED_FIELDS_DEFAULT: AiAutoGeneratedField[] = [ 'semantic', ]; -export const parseAiAutoGeneratedFieldsText = ( +export const parseAiAutoGeneratedFieldsString = ( text = AI_AUTO_GENERATED_FIELDS_DEFAULT.join(','), ): AiAutoGeneratedField[] => { const textFormatted = text.trim().toLocaleLowerCase(); diff --git a/src/site/config.ts b/src/site/config.ts index 43607ef7..bbcba05f 100644 --- a/src/site/config.ts +++ b/src/site/config.ts @@ -1,4 +1,7 @@ -import { parseAiAutoGeneratedFieldsText } from '@/photo/ai'; +import { + AI_AUTO_GENERATED_FIELDS_DEFAULT, + parseAiAutoGeneratedFieldsString, +} from '@/photo/ai'; import type { StorageType } from '@/services/storage'; import { makeUrlAbsolute, shortenUrl } from '@/utility/url'; @@ -142,7 +145,7 @@ export const CURRENT_STORAGE: StorageType = export const AI_TEXT_GENERATION_ENABLED = Boolean(process.env.OPENAI_SECRET_KEY); -export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsText( +export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsString( process.env.AI_TEXT_AUTO_GENERATED_FIELDS); // PERFORMANCE @@ -265,7 +268,7 @@ export const CONFIG_CHECKLIST_STATUS = { ? AI_TEXT_AUTO_GENERATED_FIELDS.length === 0 ? ['none'] : AI_TEXT_AUTO_GENERATED_FIELDS - : ['all'], + : AI_AUTO_GENERATED_FIELDS_DEFAULT, hasAiTextAutoGeneratedFields: Boolean(process.env.AI_TEXT_AUTO_GENERATED_FIELDS), // Performance