Merge pull request #190 from sambecker/ai-tag-refinement

Refine AI text generation

README.md

@@ -77,11 +77,11 @@ _⚠️ READ BEFORE PROCEEDING_
 3. Configure auto-generated fields (optional) 
    - Set which text fields auto-generate when uploading a photo by storing a comma-separated list, e.g., `AI_TEXT_AUTO_GENERATED_FIELDS = title, semantic`
    - Accepted values:
-     - `all` (default)
+     - `all`
-     - `title`
+     - `title` (default)
      - `caption`
-     - `tags`
+     - `tags` (default)
-     - `semantic`
+     - `semantic` (default)
      - `none`
 
 ### Web Analytics

__tests__/ai.test.ts

@@ -1,68 +1,69 @@
 /* eslint-disable quotes */
 import {
-  parseAiAutoGeneratedFieldsText,
+  AI_AUTO_GENERATED_FIELDS_DEFAULT,
+  parseAiAutoGeneratedFieldsString,
   parseTitleAndCaption,
 } from "@/photo/ai";
 
 describe('AI parses', () => {
   describe('auto-generated fields', () => {
     it('with spaces', () => {
-      expect(parseAiAutoGeneratedFieldsText())
+      expect(parseAiAutoGeneratedFieldsString())
+        .toStrictEqual(AI_AUTO_GENERATED_FIELDS_DEFAULT);
+      expect(parseAiAutoGeneratedFieldsString('all'))
         .toStrictEqual(['title', 'caption', 'tags', 'semantic']);
-      expect(parseAiAutoGeneratedFieldsText('all'))
+      expect(parseAiAutoGeneratedFieldsString('title'))
-        .toStrictEqual(['title', 'caption', 'tags', 'semantic']);
-      expect(parseAiAutoGeneratedFieldsText('title'))
         .toStrictEqual(['title']);
-      expect(parseAiAutoGeneratedFieldsText('title, caption'))
+      expect(parseAiAutoGeneratedFieldsString('title, caption'))
         .toStrictEqual(['title', 'caption']);
-      expect(parseAiAutoGeneratedFieldsText('title, caption, invalid'))
+      expect(parseAiAutoGeneratedFieldsString('title, caption, invalid'))
         .toStrictEqual(['title', 'caption']);
-      expect(parseAiAutoGeneratedFieldsText('title, caption, invalid, tags'))
+      expect(parseAiAutoGeneratedFieldsString('title, caption, invalid, tags'))
         .toStrictEqual(['title', 'caption', 'tags']);
-      expect(parseAiAutoGeneratedFieldsText('none'))
+      expect(parseAiAutoGeneratedFieldsString('none'))
         .toStrictEqual([]);
     });
     it('without spaces', () => {
-      expect(parseAiAutoGeneratedFieldsText('title,caption'))
+      expect(parseAiAutoGeneratedFieldsString('title,caption'))
         .toStrictEqual(['title', 'caption']);
-      expect(parseAiAutoGeneratedFieldsText('title,caption,invalid'))
+      expect(parseAiAutoGeneratedFieldsString('title,caption,invalid'))
         .toStrictEqual(['title', 'caption']);
-      expect(parseAiAutoGeneratedFieldsText('title,caption,invalid,tags'))
+      expect(parseAiAutoGeneratedFieldsString('title,caption,invalid,tags'))
         .toStrictEqual(['title', 'caption', 'tags']);
     });
   });
   it('received titles and captions', () => {
     // Complex case
     expect(parseTitleAndCaption(
-      `'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'`
+      `'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'`,
     )).toStrictEqual({
       title: 'Ephemeral Beauty',
       caption: 'Roses bask in fleeting sunlight',
     });
     // Without surrounding single quotes
     expect(parseTitleAndCaption(
-      `Title: "Ephemeral Beauty"  Caption: "Roses bask in fleeting sunlight."`
+      `Title: "Ephemeral Beauty"  Caption: "Roses bask in fleeting sunlight."`,
     )).toStrictEqual({
       title: 'Ephemeral Beauty',
       caption: 'Roses bask in fleeting sunlight',
     });
     // Without trailing period
     expect(parseTitleAndCaption(
-      `Title: "Ephemeral Beauty"  Caption: "Roses bask in fleeting sunlight"`
+      `Title: "Ephemeral Beauty"  Caption: "Roses bask in fleeting sunlight"`,
     )).toStrictEqual({
       title: 'Ephemeral Beauty',
       caption: 'Roses bask in fleeting sunlight',
     });
     // Without and quotes
     expect(parseTitleAndCaption(
-      `Title: Ephemeral Beauty  Caption: Roses bask in fleeting sunlight`
+      `Title: Ephemeral Beauty  Caption: Roses bask in fleeting sunlight`,
     )).toStrictEqual({
       title: 'Ephemeral Beauty',
       caption: 'Roses bask in fleeting sunlight',
     });
     // With single space
     expect(parseTitleAndCaption(
-      `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`
+      `Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`,
     )).toStrictEqual({
       title: 'Ephemeral Beauty',
       caption: 'Roses bask in fleeting sunlight',

src/photo/PhotoLarge.tsx

@@ -265,7 +265,8 @@ export default function PhotoLarge({
                           <span
                             className={clsx(
                               'text-extra-dim',
-                              'hover:underline decoration-dotted',
+                              'decoration-dotted underline-offset-[3px]',
+                              'hover:underline',
                             )}
                           >
                             {photo.focalLengthIn35MmFormatFormatted}

src/photo/actions.ts

@@ -9,6 +9,7 @@ import {
   getPhoto,
   getPhotos,
   addTagsToPhotos,
+  getUniqueTags,
 } from '@/photo/db/query';
 import { GetPhotosOptions, areOptionsSensitive } from './db';
 import {
@@ -37,7 +38,7 @@ import { blurImageFromUrl, extractImageDataFromBlobPath } from './server';
 import { TAG_FAVS, isTagFavs } from '@/tag';
 import { convertPhotoToPhotoDbInsert, Photo } from '.';
 import { runAuthenticatedAdminServerAction } from '@/auth';
-import { AI_IMAGE_QUERIES, AiImageQuery } from './ai';
+import { AiImageQuery, getAiImageQuery } from './ai';
 import { streamOpenAiImageQuery } from '@/services/openai';
 import {
   AI_TEXT_AUTO_GENERATED_FIELDS,
@@ -394,8 +395,13 @@ export const streamAiImageQueryAction = async (
   imageBase64: string,
   query: AiImageQuery,
 ) =>
-  runAuthenticatedAdminServerAction(() =>
+  runAuthenticatedAdminServerAction(async () => {
-    streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query]));
+    const existingTags = await getUniqueTags();
+    return streamOpenAiImageQuery(
+      imageBase64,
+      getAiImageQuery(query, existingTags),
+    );
+  });
 
 export const getImageBlurAction = async (url: string) =>
   runAuthenticatedAdminServerAction(() => blurImageFromUrl(url));

src/photo/ai/AiButton.tsx

@@ -1,12 +1,12 @@
 import { AiContent } from './useAiImageQueries';
 import { HiSparkles } from 'react-icons/hi';
-import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.';
+import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.';
 import { useMemo } from 'react';
 import LoaderButton from '@/components/primitives/LoaderButton';
 
 export default function AiButton({
   aiContent,
-  requestFields = ALL_AI_AUTO_GENERATED_FIELDS,
+  requestFields = AI_AUTO_GENERATED_FIELDS_ALL,
   shouldConfirm,
   className,
 }: {

src/photo/ai/index.ts

@@ -1,32 +1,40 @@
 /* eslint-disable max-len */
 
+import { Tags } from '@/tag';
+
 export type AiAutoGeneratedField =
   'title' |
   'caption' |
   'tags' |
   'semantic'
 
-export const ALL_AI_AUTO_GENERATED_FIELDS: AiAutoGeneratedField[] = [
+export const AI_AUTO_GENERATED_FIELDS_ALL: AiAutoGeneratedField[] = [
   'title',
   'caption',
   'tags',
   'semantic',
 ];
 
-export const parseAiAutoGeneratedFieldsText = (
+export const AI_AUTO_GENERATED_FIELDS_DEFAULT: AiAutoGeneratedField[] = [
-  text = 'all',
+  'title',
+  'tags',
+  'semantic',
+];
+
+export const parseAiAutoGeneratedFieldsString = (
+  text = AI_AUTO_GENERATED_FIELDS_DEFAULT.join(','),
 ): AiAutoGeneratedField[] => {
   const textFormatted = text.trim().toLocaleLowerCase();
   if (textFormatted === 'none') {
     return [];
   } else if (textFormatted === 'all') {
-    return ALL_AI_AUTO_GENERATED_FIELDS;
+    return AI_AUTO_GENERATED_FIELDS_ALL;
   } else {
     const fields = textFormatted
       .toLocaleLowerCase()
       .split(',')
       .map(field => field.trim())
-      .filter(field => ALL_AI_AUTO_GENERATED_FIELDS
+      .filter(field => AI_AUTO_GENERATED_FIELDS_ALL
         .includes(field as AiAutoGeneratedField));
     return fields as AiAutoGeneratedField[];
   }
@@ -42,15 +50,25 @@ export type AiImageQuery =
   'description-large' |
   'description-semantic';
 
-export const AI_IMAGE_QUERIES: Record<AiImageQuery, string> = {
+export const getAiImageQuery = (
-  'title': 'Write a compelling title for this image in 3 words or less',
+  query: AiImageQuery,
-  'caption': 'Write a pithy caption for this image in 6 words or less and no punctuation',
+  existingTags: Tags = [],
-  'title-and-caption': 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"',
+): string => {
-  'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs',
+  switch (query) {
-  'description-small': 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"',
+  case 'title': return 'Write a compelling title for this image in 3 words or less';
-  'description': 'Describe this image',
+  case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation';
-  'description-large': 'Describe this image in detail',
+  case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"';
-  'description-semantic': 'List up to 5 things in this image without description as a comma-separated list',
+  case 'tags':
+    const tagQuery = 'Describe this image in 1-2 comma-separated unique keywords, with no adjective or adverbs. Avoid using general terms like "nature," "travel," "architecture," or "sky." Use terms that are highly specific to the image and not redundant.';
+    const tags = existingTags.map(({ tag }) => tag).join(', ');
+    return tags
+      ? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.`
+      : tagQuery;
+  case 'description-small': return 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"';
+  case 'description': return 'Describe this image';
+  case 'description-large': return 'Describe this image in detail';
+  case 'description-semantic': return 'List up to 5 things in this image without description as a comma-separated list';
+  }
 };
 
 export const parseTitleAndCaption = (text: string) => {

src/photo/ai/server.ts

@@ -1,9 +1,10 @@
 import { generateOpenAiImageQuery } from '@/services/openai';
 import {
-  AI_IMAGE_QUERIES,
   AiAutoGeneratedField,
+  getAiImageQuery,
   parseTitleAndCaption,
 } from '.';
+import { getUniqueTags } from '../db/query';
 
 export const generateAiImageQueries = async (
   imageBase64?: string,
@@ -29,7 +30,7 @@ export const generateAiImageQueries = async (
       ) {
         const titleAndCaption = await generateOpenAiImageQuery(
           imageBase64,
-          AI_IMAGE_QUERIES['title-and-caption'],
+          getAiImageQuery('title-and-caption'),
         );
         if (titleAndCaption) {
           const titleAndCaptionParsed = parseTitleAndCaption(titleAndCaption);
@@ -40,28 +41,29 @@ export const generateAiImageQueries = async (
         if (textFieldsToGenerate.includes('title')) {
           title = await generateOpenAiImageQuery(
             imageBase64,
-            AI_IMAGE_QUERIES['title'],
+            getAiImageQuery('title'),
           );
         }
         if (textFieldsToGenerate.includes('caption')) {
           caption =  await generateOpenAiImageQuery(
             imageBase64,
-            AI_IMAGE_QUERIES['caption'],
+            getAiImageQuery('caption'),
           );
         }
       }
   
       if (textFieldsToGenerate.includes('tags')) {
+        const existingTags = await getUniqueTags();
         tags = await generateOpenAiImageQuery(
           imageBase64,
-          AI_IMAGE_QUERIES['tags'],
+          getAiImageQuery('tags', existingTags),
         );
       }
   
       if (textFieldsToGenerate.includes('semantic')) {
         semanticDescription = await generateOpenAiImageQuery(
           imageBase64,
-          AI_IMAGE_QUERIES['description-small'],
+          getAiImageQuery('description-small'),
         );
       }
     }

src/photo/ai/useAiImageQueries.ts

@@ -1,7 +1,7 @@
 import { useCallback, useEffect, useRef } from 'react';
 import useAiImageQuery from './useAiImageQuery';
 import useTitleCaptionAiImageQuery from './useTitleCaptionAiImageQuery';
-import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.';
+import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.';
 
 export type AiContent = ReturnType<typeof useAiImageQueries>;
 
@@ -59,7 +59,7 @@ export default function useAiImageQueries(
   const hasRunAllQueriesOnce = useRef(false);
 
   const request = useCallback(async (
-    fields = ALL_AI_AUTO_GENERATED_FIELDS,
+    fields = AI_AUTO_GENERATED_FIELDS_ALL,
   ) => {
     if (process.env.NODE_ENV !== 'production') {
       console.log('RUNNING AI QUERIES', fields);

src/services/openai.ts

@@ -73,7 +73,7 @@ export const streamOpenAiImageQuery = async (
 
   if (args) {
     (async () => {
-      const { textStream } = await streamText(args);
+      const { textStream } = streamText(args);
       for await (const delta of textStream) {
         stream.update(cleanUpAiTextResponse(delta));
       }

src/site/SiteChecklistClient.tsx

@@ -424,7 +424,9 @@ export default function SiteChecklistClient({
             >
               Comma-separated fields to auto-generate when
               uploading photos. Accepted values: title, caption,
-              tags, description, all, or none (default is {'"all"'}):
+              tags, description, all, or none
+              {' '}
+              (default: {'"title, tags, semantic"'}):
               {renderEnvVars(['AI_TEXT_AUTO_GENERATED_FIELDS'])}
             </ChecklistRow>
           </Checklist>

src/site/config.ts

@@ -1,4 +1,7 @@
-import { parseAiAutoGeneratedFieldsText } from '@/photo/ai';
+import { 
+  AI_AUTO_GENERATED_FIELDS_DEFAULT,
+  parseAiAutoGeneratedFieldsString,
+} from '@/photo/ai';
 import type { StorageType } from '@/services/storage';
 import { makeUrlAbsolute, shortenUrl } from '@/utility/url';
 
@@ -142,7 +145,7 @@ export const CURRENT_STORAGE: StorageType =
 
 export const AI_TEXT_GENERATION_ENABLED =
   Boolean(process.env.OPENAI_SECRET_KEY);
-export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsText(
+export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsString(
   process.env.AI_TEXT_AUTO_GENERATED_FIELDS);
 
 // PERFORMANCE
@@ -265,7 +268,7 @@ export const CONFIG_CHECKLIST_STATUS = {
     ? AI_TEXT_AUTO_GENERATED_FIELDS.length === 0
       ? ['none']
       : AI_TEXT_AUTO_GENERATED_FIELDS
-    : ['all'],
+    : AI_AUTO_GENERATED_FIELDS_DEFAULT,
   hasAiTextAutoGeneratedFields:
     Boolean(process.env.AI_TEXT_AUTO_GENERATED_FIELDS),
   // Performance