From e2e8c8edda343f8bfbb4306d17929ea6c2d769a9 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 23:05:21 -0500 Subject: [PATCH] Wire up page-level AI streaming --- src/photo/PhotoEditPageClient.tsx | 36 +++-- src/photo/actions.ts | 8 +- src/photo/ai/index.ts | 38 +++-- src/photo/ai/useImageQueries.ts | 55 +++++++ src/photo/ai/useImageQuery.ts | 10 +- src/photo/ai/useTitleCaptionImageQuery.ts | 32 ++++ src/photo/form/PhotoForm.tsx | 173 ++-------------------- src/photo/form/index.ts | 12 +- 8 files changed, 157 insertions(+), 207 deletions(-) create mode 100644 src/photo/ai/useImageQueries.ts create mode 100644 src/photo/ai/useTitleCaptionImageQuery.ts diff --git a/src/photo/PhotoEditPageClient.tsx b/src/photo/PhotoEditPageClient.tsx index a26c054d..ff302454 100644 --- a/src/photo/PhotoEditPageClient.tsx +++ b/src/photo/PhotoEditPageClient.tsx @@ -12,6 +12,9 @@ import IconGrSync from '@/site/IconGrSync'; import { getExifDataAction } from './actions'; import { Tags } from '@/tag'; import { useState } from 'react'; +import useImageQueries from './ai/useImageQueries'; +import { HiSparkles } from 'react-icons/hi'; +import Spinner from '@/components/Spinner'; export default function PhotoEditPageClient({ photo, @@ -37,6 +40,8 @@ export default function PhotoEditPageClient({ seedExifData, ); + const aiContent = useImageQueries(); + return ( - - } +
+ +
+ + } + > + EXIF + +
+
} isLoading={pending} > diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 759e2641..f75855aa 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -34,7 +34,7 @@ import { extractExifDataFromBlobPath } from './server'; import { TAG_FAVS, isTagFavs } from '@/tag'; import { convertPhotoToPhotoDbInsert } from '.'; import { safelyRunAdminServerAction } from '@/auth'; -import { ImageQuery, streamImageQuery } from './ai'; +import { AiImageQuery, streamAiImageQuery } from './ai'; export async function createPhotoAction(formData: FormData) { return safelyRunAdminServerAction(async () => { @@ -183,10 +183,10 @@ export async function syncCacheAction() { return safelyRunAdminServerAction(revalidateAllKeysAndPaths); } -export async function streamImageQueryAction( +export async function streamAiImageQueryAction( imageBase64: string, - query: ImageQuery, + query: AiImageQuery, ) { return safelyRunAdminServerAction(async () => - streamImageQuery(imageBase64, query)); + streamAiImageQuery(imageBase64, query)); } diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index b8375008..a2bb8669 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -1,29 +1,27 @@ +/* eslint-disable max-len */ + import { streamOpenAiImageQuery } from '@/services/openai'; -export type ImageQuery = +export type AiImageQuery = 'title' | 'caption' | + 'title-and-caption' | 'tags' | - 'descriptionSmall' | - 'descriptionMedium' | - 'descriptionLarge' | - 'rich' | + 'description-small' | + 'description' | + 'description-large' | 'semantic'; -export const IMAGE_QUERIES: Record = { - // title: 'Provide a short title for this image', - title: 'Provide a short title for this image in 3 words or less', - caption: 'What is a pithy caption for this image in 8 words or less?', - // eslint-disable-next-line max-len - tags: 'Describe this image three or less comma-separated keywords with no adjective or adverbs', - descriptionSmall: 'Describe this image succinctly', - descriptionMedium: 'Describe this image', - descriptionLarge: 'Describe this image in detail', - // eslint-disable-next-line max-len - rich: 'What is a short title and pithy caption of 8 words or less for this image?', - // eslint-disable-next-line max-len - semantic: 'List up to 5 things in this image without description as a comma-separated list', +export const AI_IMAGE_QUERIES: Record = { + 'title': 'Provide a short title for this image in 3 words or less', + 'caption': 'What is a pithy caption for this image in 8 words or less?', + 'title-and-caption': 'Write a short title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"', + 'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs', + 'description-small': 'Describe this image succinctly', + 'description': 'Describe this image', + 'description-large': 'Describe this image in detail', + 'semantic': 'List up to 5 things in this image without description as a comma-separated list', }; -export const streamImageQuery = (imageBase64: string, query: ImageQuery) => - streamOpenAiImageQuery(imageBase64, IMAGE_QUERIES[query]); +export const streamAiImageQuery = (imageBase64: string, query: AiImageQuery) => + streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query]); diff --git a/src/photo/ai/useImageQueries.ts b/src/photo/ai/useImageQueries.ts new file mode 100644 index 00000000..1fffe33a --- /dev/null +++ b/src/photo/ai/useImageQueries.ts @@ -0,0 +1,55 @@ +import { useCallback, useState } from 'react'; +import useImageQuery from './useImageQuery'; +import useTitleCaptionImageQuery from './useTitleCaptionImageQuery'; + +export type AiContent = ReturnType; + +export default function useImageQueries() { + const [imageData, setImageData] = useState(); + + const isReady = Boolean(imageData); + + const [ + requestTitleCaption, + title, + caption, + isLoadingTitleCaption, + ] = useTitleCaptionImageQuery(imageData); + + const [ + requestTags, + tags, + isLoadingTags, + ] = useImageQuery(imageData, 'tags'); + + const [ + requestSemantic, + semantic, + isLoadingSemantic, + ] = useImageQuery(imageData, 'semantic'); + + const isLoading = isLoadingTitleCaption || isLoadingTags || isLoadingSemantic; + + const request = useCallback(async () => { + if (!isLoading) { + console.log('REQUESTING ALL IMAGE QUERIES'); + requestTitleCaption(); + requestTags(); + requestSemantic(); + } + }, [isLoading, requestTitleCaption, requestTags, requestSemantic]); + + return { + request, + title, + caption, + tags, + semantic, + isReady, + isLoading, + isLoadingTitleCaption, + isLoadingTags, + isLoadingSemantic, + setImageData, + }; +} diff --git a/src/photo/ai/useImageQuery.ts b/src/photo/ai/useImageQuery.ts index 9f0079eb..6eb598ef 100644 --- a/src/photo/ai/useImageQuery.ts +++ b/src/photo/ai/useImageQuery.ts @@ -1,11 +1,11 @@ import { useCallback, useState } from 'react'; -import { streamImageQueryAction } from '../actions'; +import { streamAiImageQueryAction } from '../actions'; import { readStreamableValue } from 'ai/rsc'; -import { ImageQuery } from '.'; +import { AiImageQuery } from '.'; export default function useImageQuery( imageBase64: string | undefined, - query: ImageQuery, + query: AiImageQuery, ) { const [text, setText] = useState(''); const [error, setError] = useState(); @@ -15,12 +15,12 @@ export default function useImageQuery( if (imageBase64) { setIsLoading(true); try { - const textStream = await streamImageQueryAction( + const textStream = await streamAiImageQueryAction( imageBase64, query, ); for await (const text of readStreamableValue(textStream)) { - setText(text ?? ''); + setText((text ?? '').replaceAll('\n', ' ')); } setIsLoading(false); } catch (e) { diff --git a/src/photo/ai/useTitleCaptionImageQuery.ts b/src/photo/ai/useTitleCaptionImageQuery.ts new file mode 100644 index 00000000..c9263fae --- /dev/null +++ b/src/photo/ai/useTitleCaptionImageQuery.ts @@ -0,0 +1,32 @@ +import { useMemo } from 'react'; +import useImageQuery from './useImageQuery'; + +export default function useTitleCaptionImageQuery( + imageBase64: string | undefined, +) { + const [ + request, + text, + isLoading, + error, + ] = useImageQuery(imageBase64, 'title-and-caption'); + + const { title, caption } = useMemo(() => { + const matches = text.includes('Title') + ? text.match(/^[`']*Title: "*(.*?)\.*"* Caption: "*(.*?)\.*"*[`']*$/) + : text.match(/^(.*?): (.*?)$/); + + return { + title: matches?.[1] ?? '', + caption: matches?.[2] ?? '', + }; + }, [text]); + + return [ + request, + title, + caption, + isLoading, + error, + ] as const; +} diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 0ff11a6e..c2708e51 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -25,8 +25,7 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; -import Spinner from '@/components/Spinner'; -import useImageQuery from '../ai/useImageQuery'; +import { AiContent } from '../ai/useImageQueries'; const THUMBNAIL_SIZE = 300; @@ -35,7 +34,7 @@ export default function PhotoForm({ updatedExifData, type = 'create', uniqueTags, - aiTextGeneration, + aiContent, debugBlur, onTitleChange, onFormStatusChange, @@ -44,7 +43,8 @@ export default function PhotoForm({ updatedExifData?: Partial type?: 'create' | 'edit' uniqueTags?: Tags - aiTextGeneration?: boolean + aiContent?: AiContent + setImageData?: (imageData: string) => void debugBlur?: boolean onTitleChange?: (updatedTitle: string) => void onFormStatusChange?: (pending: boolean) => void @@ -55,8 +55,6 @@ export default function PhotoForm({ useState(getFormErrors(initialPhotoForm)); const [blurError, setBlurError] = useState(); - const [imageData, setImageData] = - useState(); // Update form when EXIF data // is refreshed by parent @@ -122,123 +120,12 @@ export default function PhotoForm({ } }, []); - // const [ - // requestTitle, - // title, - // isLoadingTitle, - // errorTitle, - // ] = useImageQuery(imageData, 'title'); - - // const [ - // requestCaption, - // caption, - // isLoadingCaption, - // errorCaption, - // ] = useImageQuery(imageData, 'caption'); - - const [ - requestTags, - tags, - isLoadingTags, - errorTags, - ] = useImageQuery(imageData, 'tags'); - - const [ - requestRich, - rich, - isLoadingRich, - errorRich, - ] = useImageQuery(imageData, 'rich'); - - // const [ - // requestDescriptionSmall, - // descriptionSmall, - // isLoadingDescriptionSmall, - // errorDescriptionSmall, - // ] = useImageQuery(imageData, 'descriptionSmall'); - - const [ - requestSemantic, - semantic, - isLoadingSemantic, - errorSemantic, - ] = useImageQuery(imageData, 'semantic'); - - const renderAiButton = ( - label: string, - onClick: () => void, - isLoading: boolean, - error?: any, - ) => - ; - return (
- {blurError && + {debugBlur && blurError &&
{blurError}
} -
- {/* {renderAiButton( - 'Title', - requestTitle, - isLoadingTitle, - errorTitle, - )} - {renderAiButton( - 'Caption', - requestCaption, - isLoadingCaption, - errorCaption, - )} - {renderAiButton( - 'Tags', - requestTags, - isLoadingTags, - errorTags, - )} */} - {renderAiButton( - 'Rich', - requestRich, - isLoadingRich, - errorRich, - )} - {renderAiButton( - 'Tags', - requestTags, - isLoadingTags, - errorTags, - )} - {renderAiButton( - 'Semantic', - requestSemantic, - isLoadingSemantic, - errorSemantic, - )} - {/* {renderAiButton( - 'Description', - requestDescriptionSmall, - isLoadingDescriptionSmall, - errorDescriptionSmall, - )} */} -
@@ -271,48 +158,10 @@ export default function PhotoForm({ height={height} />}
- {/*

- ✨ TITLE: {title} {isLoadingTitle && <> - - - - } -

-

- ✨ CAPTION: {caption} {isLoadingCaption && <> - - - - } -

*/} -

- ✨ RICH: {rich} {isLoadingRich && <> - - - - } -

-

- ✨ TAGS: {tags} {isLoadingTags && <> - - - - } -

-

- ✨ SEMANTIC: {semantic} {isLoadingSemantic && <> - - - - } -

- {/*

- ✨ DESCRIPTION: {descriptionSmall} {isLoadingDescriptionSmall && <> - - - - } -

*/} +
Title: {aiContent?.title}
+
Caption: {aiContent?.caption}
+
Tags: {aiContent?.tags}
+
Semantic: {aiContent?.semantic}
blur()} @@ -325,7 +174,7 @@ export default function PhotoForm({ annotation: formatCount(count), annotationAria: formatCountDescriptive(count, 'tagged'), })), - aiTextGeneration, + aiContent !== undefined, ) .map(([key, { label, diff --git a/src/photo/form/index.ts b/src/photo/form/index.ts index 39ec8661..0e4a05cd 100644 --- a/src/photo/form/index.ts +++ b/src/photo/form/index.ts @@ -72,12 +72,6 @@ const FORM_METADATA = ( validateStringMaxLength: STRING_MAX_LENGTH_LONG, shouldHide: ({ title, caption }) => !title && !caption, }, - semanticDescription: { - label: 'semantic description', - capitalize: true, - validateStringMaxLength: STRING_MAX_LENGTH_LONG, - hide: !aiTextGeneration, - }, tags: { label: 'tags', tagOptions, @@ -85,6 +79,12 @@ const FORM_METADATA = ( ? `'${TAG_FAVS}' is a reserved tag` : undefined, }, + semanticDescription: { + label: 'semantic description', + capitalize: true, + validateStringMaxLength: STRING_MAX_LENGTH_LONG, + hide: !aiTextGeneration, + }, id: { label: 'id', readOnly: true, hideIfEmpty: true }, blurData: { label: 'blur data',