Wire up page-level AI streaming
This commit is contained in:
Sam Becker
parent
1371a8dcc4
commit
e2e8c8edda
@ -12,6 +12,9 @@ import IconGrSync from '@/site/IconGrSync';
|
||||
import { getExifDataAction } from './actions';
|
||||
import { Tags } from '@/tag';
|
||||
import { useState } from 'react';
|
||||
import useImageQueries from './ai/useImageQueries';
|
||||
import { HiSparkles } from 'react-icons/hi';
|
||||
import Spinner from '@/components/Spinner';
|
||||
|
||||
export default function PhotoEditPageClient({
|
||||
photo,
|
||||
@ -37,6 +40,8 @@ export default function PhotoEditPageClient({
|
||||
seedExifData,
|
||||
);
|
||||
|
||||
const aiContent = useImageQueries();
|
||||
|
||||
return (
|
||||
<AdminChildPage
|
||||
backPath={PATH_ADMIN_PHOTOS}
|
||||
@ -45,6 +50,16 @@ export default function PhotoEditPageClient({
|
||||
? updatedTitle
|
||||
: photo.title || photo.id}
|
||||
accessory={
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
className="min-w-[3.25rem] flex justify-center"
|
||||
onClick={aiContent.request}
|
||||
disabled={!aiContent.isReady || aiContent.isLoading}
|
||||
>
|
||||
{aiContent.isLoading
|
||||
? <Spinner />
|
||||
: <HiSparkles size={16} />}
|
||||
</button>
|
||||
<form action={action}>
|
||||
<input name="photoUrl" value={photo.url} hidden readOnly />
|
||||
<SubmitButtonWithStatus
|
||||
@ -54,7 +69,8 @@ export default function PhotoEditPageClient({
|
||||
>
|
||||
EXIF
|
||||
</SubmitButtonWithStatus>
|
||||
</form>}
|
||||
</form>
|
||||
</div>}
|
||||
isLoading={pending}
|
||||
>
|
||||
<PhotoForm
|
||||
@ -64,7 +80,7 @@ export default function PhotoEditPageClient({
|
||||
? updatedExifData
|
||||
: undefined}
|
||||
uniqueTags={uniqueTags}
|
||||
aiTextGeneration={aiTextGeneration}
|
||||
aiContent={aiTextGeneration ? aiContent : undefined}
|
||||
onTitleChange={setUpdatedTitle}
|
||||
onFormStatusChange={setIsPending}
|
||||
/>
|
||||
|
||||
@ -34,7 +34,7 @@ import { extractExifDataFromBlobPath } from './server';
|
||||
import { TAG_FAVS, isTagFavs } from '@/tag';
|
||||
import { convertPhotoToPhotoDbInsert } from '.';
|
||||
import { safelyRunAdminServerAction } from '@/auth';
|
||||
import { ImageQuery, streamImageQuery } from './ai';
|
||||
import { AiImageQuery, streamAiImageQuery } from './ai';
|
||||
|
||||
export async function createPhotoAction(formData: FormData) {
|
||||
return safelyRunAdminServerAction(async () => {
|
||||
@ -183,10 +183,10 @@ export async function syncCacheAction() {
|
||||
return safelyRunAdminServerAction(revalidateAllKeysAndPaths);
|
||||
}
|
||||
|
||||
export async function streamImageQueryAction(
|
||||
export async function streamAiImageQueryAction(
|
||||
imageBase64: string,
|
||||
query: ImageQuery,
|
||||
query: AiImageQuery,
|
||||
) {
|
||||
return safelyRunAdminServerAction(async () =>
|
||||
streamImageQuery(imageBase64, query));
|
||||
streamAiImageQuery(imageBase64, query));
|
||||
}
|
||||
|
||||
@ -1,29 +1,27 @@
|
||||
/* eslint-disable max-len */
|
||||
|
||||
import { streamOpenAiImageQuery } from '@/services/openai';
|
||||
|
||||
export type ImageQuery =
|
||||
export type AiImageQuery =
|
||||
'title' |
|
||||
'caption' |
|
||||
'title-and-caption' |
|
||||
'tags' |
|
||||
'descriptionSmall' |
|
||||
'descriptionMedium' |
|
||||
'descriptionLarge' |
|
||||
'rich' |
|
||||
'description-small' |
|
||||
'description' |
|
||||
'description-large' |
|
||||
'semantic';
|
||||
|
||||
export const IMAGE_QUERIES: Record<ImageQuery, string> = {
|
||||
// title: 'Provide a short title for this image',
|
||||
title: 'Provide a short title for this image in 3 words or less',
|
||||
caption: 'What is a pithy caption for this image in 8 words or less?',
|
||||
// eslint-disable-next-line max-len
|
||||
tags: 'Describe this image three or less comma-separated keywords with no adjective or adverbs',
|
||||
descriptionSmall: 'Describe this image succinctly',
|
||||
descriptionMedium: 'Describe this image',
|
||||
descriptionLarge: 'Describe this image in detail',
|
||||
// eslint-disable-next-line max-len
|
||||
rich: 'What is a short title and pithy caption of 8 words or less for this image?',
|
||||
// eslint-disable-next-line max-len
|
||||
semantic: 'List up to 5 things in this image without description as a comma-separated list',
|
||||
export const AI_IMAGE_QUERIES: Record<AiImageQuery, string> = {
|
||||
'title': 'Provide a short title for this image in 3 words or less',
|
||||
'caption': 'What is a pithy caption for this image in 8 words or less?',
|
||||
'title-and-caption': 'Write a short title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"',
|
||||
'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs',
|
||||
'description-small': 'Describe this image succinctly',
|
||||
'description': 'Describe this image',
|
||||
'description-large': 'Describe this image in detail',
|
||||
'semantic': 'List up to 5 things in this image without description as a comma-separated list',
|
||||
};
|
||||
|
||||
export const streamImageQuery = (imageBase64: string, query: ImageQuery) =>
|
||||
streamOpenAiImageQuery(imageBase64, IMAGE_QUERIES[query]);
|
||||
export const streamAiImageQuery = (imageBase64: string, query: AiImageQuery) =>
|
||||
streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query]);
|
||||
|
||||
55
src/photo/ai/useImageQueries.ts
Normal file
55
src/photo/ai/useImageQueries.ts
Normal file
@ -0,0 +1,55 @@
|
||||
import { useCallback, useState } from 'react';
|
||||
import useImageQuery from './useImageQuery';
|
||||
import useTitleCaptionImageQuery from './useTitleCaptionImageQuery';
|
||||
|
||||
export type AiContent = ReturnType<typeof useImageQueries>;
|
||||
|
||||
export default function useImageQueries() {
|
||||
const [imageData, setImageData] = useState<string>();
|
||||
|
||||
const isReady = Boolean(imageData);
|
||||
|
||||
const [
|
||||
requestTitleCaption,
|
||||
title,
|
||||
caption,
|
||||
isLoadingTitleCaption,
|
||||
] = useTitleCaptionImageQuery(imageData);
|
||||
|
||||
const [
|
||||
requestTags,
|
||||
tags,
|
||||
isLoadingTags,
|
||||
] = useImageQuery(imageData, 'tags');
|
||||
|
||||
const [
|
||||
requestSemantic,
|
||||
semantic,
|
||||
isLoadingSemantic,
|
||||
] = useImageQuery(imageData, 'semantic');
|
||||
|
||||
const isLoading = isLoadingTitleCaption || isLoadingTags || isLoadingSemantic;
|
||||
|
||||
const request = useCallback(async () => {
|
||||
if (!isLoading) {
|
||||
console.log('REQUESTING ALL IMAGE QUERIES');
|
||||
requestTitleCaption();
|
||||
requestTags();
|
||||
requestSemantic();
|
||||
}
|
||||
}, [isLoading, requestTitleCaption, requestTags, requestSemantic]);
|
||||
|
||||
return {
|
||||
request,
|
||||
title,
|
||||
caption,
|
||||
tags,
|
||||
semantic,
|
||||
isReady,
|
||||
isLoading,
|
||||
isLoadingTitleCaption,
|
||||
isLoadingTags,
|
||||
isLoadingSemantic,
|
||||
setImageData,
|
||||
};
|
||||
}
|
||||
@ -1,11 +1,11 @@
|
||||
import { useCallback, useState } from 'react';
|
||||
import { streamImageQueryAction } from '../actions';
|
||||
import { streamAiImageQueryAction } from '../actions';
|
||||
import { readStreamableValue } from 'ai/rsc';
|
||||
import { ImageQuery } from '.';
|
||||
import { AiImageQuery } from '.';
|
||||
|
||||
export default function useImageQuery(
|
||||
imageBase64: string | undefined,
|
||||
query: ImageQuery,
|
||||
query: AiImageQuery,
|
||||
) {
|
||||
const [text, setText] = useState('');
|
||||
const [error, setError] = useState<any>();
|
||||
@ -15,12 +15,12 @@ export default function useImageQuery(
|
||||
if (imageBase64) {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const textStream = await streamImageQueryAction(
|
||||
const textStream = await streamAiImageQueryAction(
|
||||
imageBase64,
|
||||
query,
|
||||
);
|
||||
for await (const text of readStreamableValue(textStream)) {
|
||||
setText(text ?? '');
|
||||
setText((text ?? '').replaceAll('\n', ' '));
|
||||
}
|
||||
setIsLoading(false);
|
||||
} catch (e) {
|
||||
|
||||
32
src/photo/ai/useTitleCaptionImageQuery.ts
Normal file
32
src/photo/ai/useTitleCaptionImageQuery.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import { useMemo } from 'react';
|
||||
import useImageQuery from './useImageQuery';
|
||||
|
||||
export default function useTitleCaptionImageQuery(
|
||||
imageBase64: string | undefined,
|
||||
) {
|
||||
const [
|
||||
request,
|
||||
text,
|
||||
isLoading,
|
||||
error,
|
||||
] = useImageQuery(imageBase64, 'title-and-caption');
|
||||
|
||||
const { title, caption } = useMemo(() => {
|
||||
const matches = text.includes('Title')
|
||||
? text.match(/^[`']*Title: "*(.*?)\.*"* Caption: "*(.*?)\.*"*[`']*$/)
|
||||
: text.match(/^(.*?): (.*?)$/);
|
||||
|
||||
return {
|
||||
title: matches?.[1] ?? '',
|
||||
caption: matches?.[2] ?? '',
|
||||
};
|
||||
}, [text]);
|
||||
|
||||
return [
|
||||
request,
|
||||
title,
|
||||
caption,
|
||||
isLoading,
|
||||
error,
|
||||
] as const;
|
||||
}
|
||||
@ -25,8 +25,7 @@ import ImageBlurFallback from '@/components/ImageBlurFallback';
|
||||
import { BLUR_ENABLED } from '@/site/config';
|
||||
import { Tags, sortTagsObjectWithoutFavs } from '@/tag';
|
||||
import { formatCount, formatCountDescriptive } from '@/utility/string';
|
||||
import Spinner from '@/components/Spinner';
|
||||
import useImageQuery from '../ai/useImageQuery';
|
||||
import { AiContent } from '../ai/useImageQueries';
|
||||
|
||||
const THUMBNAIL_SIZE = 300;
|
||||
|
||||
@ -35,7 +34,7 @@ export default function PhotoForm({
|
||||
updatedExifData,
|
||||
type = 'create',
|
||||
uniqueTags,
|
||||
aiTextGeneration,
|
||||
aiContent,
|
||||
debugBlur,
|
||||
onTitleChange,
|
||||
onFormStatusChange,
|
||||
@ -44,7 +43,8 @@ export default function PhotoForm({
|
||||
updatedExifData?: Partial<PhotoFormData>
|
||||
type?: 'create' | 'edit'
|
||||
uniqueTags?: Tags
|
||||
aiTextGeneration?: boolean
|
||||
aiContent?: AiContent
|
||||
setImageData?: (imageData: string) => void
|
||||
debugBlur?: boolean
|
||||
onTitleChange?: (updatedTitle: string) => void
|
||||
onFormStatusChange?: (pending: boolean) => void
|
||||
@ -55,8 +55,6 @@ export default function PhotoForm({
|
||||
useState(getFormErrors(initialPhotoForm));
|
||||
const [blurError, setBlurError] =
|
||||
useState<string>();
|
||||
const [imageData, setImageData] =
|
||||
useState<string>();
|
||||
|
||||
// Update form when EXIF data
|
||||
// is refreshed by parent
|
||||
@ -122,123 +120,12 @@ export default function PhotoForm({
|
||||
}
|
||||
}, []);
|
||||
|
||||
// const [
|
||||
// requestTitle,
|
||||
// title,
|
||||
// isLoadingTitle,
|
||||
// errorTitle,
|
||||
// ] = useImageQuery(imageData, 'title');
|
||||
|
||||
// const [
|
||||
// requestCaption,
|
||||
// caption,
|
||||
// isLoadingCaption,
|
||||
// errorCaption,
|
||||
// ] = useImageQuery(imageData, 'caption');
|
||||
|
||||
const [
|
||||
requestTags,
|
||||
tags,
|
||||
isLoadingTags,
|
||||
errorTags,
|
||||
] = useImageQuery(imageData, 'tags');
|
||||
|
||||
const [
|
||||
requestRich,
|
||||
rich,
|
||||
isLoadingRich,
|
||||
errorRich,
|
||||
] = useImageQuery(imageData, 'rich');
|
||||
|
||||
// const [
|
||||
// requestDescriptionSmall,
|
||||
// descriptionSmall,
|
||||
// isLoadingDescriptionSmall,
|
||||
// errorDescriptionSmall,
|
||||
// ] = useImageQuery(imageData, 'descriptionSmall');
|
||||
|
||||
const [
|
||||
requestSemantic,
|
||||
semantic,
|
||||
isLoadingSemantic,
|
||||
errorSemantic,
|
||||
] = useImageQuery(imageData, 'semantic');
|
||||
|
||||
const renderAiButton = (
|
||||
label: string,
|
||||
onClick: () => void,
|
||||
isLoading: boolean,
|
||||
error?: any,
|
||||
) =>
|
||||
<button
|
||||
onClick={onClick}
|
||||
disabled={!imageData || isLoading}
|
||||
className={clsx(
|
||||
'flex gap-2 items-center justify-center',
|
||||
'disabled:opacity-50 text-sm px-2.5 min-h-0 py-1.5',
|
||||
Boolean(error) && 'error text-error',
|
||||
)}
|
||||
>
|
||||
<span>
|
||||
{label}
|
||||
</span>
|
||||
<span className="min-w-4">
|
||||
{isLoading
|
||||
? <Spinner className="translate-y-[1.5px]" />
|
||||
: <>✨</>}
|
||||
</span>
|
||||
</button>;
|
||||
|
||||
return (
|
||||
<div className="space-y-8 max-w-[38rem]">
|
||||
{blurError &&
|
||||
{debugBlur && blurError &&
|
||||
<div className="border error text-error rounded-md px-2 py-1">
|
||||
{blurError}
|
||||
</div>}
|
||||
<div className="flex gap-2 flex-wrap">
|
||||
{/* {renderAiButton(
|
||||
'Title',
|
||||
requestTitle,
|
||||
isLoadingTitle,
|
||||
errorTitle,
|
||||
)}
|
||||
{renderAiButton(
|
||||
'Caption',
|
||||
requestCaption,
|
||||
isLoadingCaption,
|
||||
errorCaption,
|
||||
)}
|
||||
{renderAiButton(
|
||||
'Tags',
|
||||
requestTags,
|
||||
isLoadingTags,
|
||||
errorTags,
|
||||
)} */}
|
||||
{renderAiButton(
|
||||
'Rich',
|
||||
requestRich,
|
||||
isLoadingRich,
|
||||
errorRich,
|
||||
)}
|
||||
{renderAiButton(
|
||||
'Tags',
|
||||
requestTags,
|
||||
isLoadingTags,
|
||||
errorTags,
|
||||
)}
|
||||
{renderAiButton(
|
||||
'Semantic',
|
||||
requestSemantic,
|
||||
isLoadingSemantic,
|
||||
errorSemantic,
|
||||
)}
|
||||
{/* {renderAiButton(
|
||||
'Description',
|
||||
requestDescriptionSmall,
|
||||
isLoadingDescriptionSmall,
|
||||
errorDescriptionSmall,
|
||||
)} */}
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<ImageBlurFallback
|
||||
alt="Upload"
|
||||
@ -255,7 +142,7 @@ export default function PhotoForm({
|
||||
imageUrl={url}
|
||||
width={width}
|
||||
height={height}
|
||||
onLoad={setImageData}
|
||||
onLoad={aiContent?.setImageData}
|
||||
onCapture={updateBlurData}
|
||||
onError={setBlurError}
|
||||
/>
|
||||
@ -271,48 +158,10 @@ export default function PhotoForm({
|
||||
height={height}
|
||||
/>}
|
||||
</div>
|
||||
{/* <p>
|
||||
✨ TITLE: {title} {isLoadingTitle && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p>
|
||||
<p>
|
||||
✨ CAPTION: {caption} {isLoadingCaption && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p> */}
|
||||
<p>
|
||||
✨ RICH: {rich} {isLoadingRich && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p>
|
||||
<p>
|
||||
✨ TAGS: {tags} {isLoadingTags && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p>
|
||||
<p>
|
||||
✨ SEMANTIC: {semantic} {isLoadingSemantic && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p>
|
||||
{/* <p>
|
||||
✨ DESCRIPTION: {descriptionSmall} {isLoadingDescriptionSmall && <>
|
||||
<span className="inline-flex translate-y-[1.5px]">
|
||||
<Spinner />
|
||||
</span>
|
||||
</>}
|
||||
</p> */}
|
||||
<div>Title: {aiContent?.title}</div>
|
||||
<div>Caption: {aiContent?.caption}</div>
|
||||
<div>Tags: {aiContent?.tags}</div>
|
||||
<div>Semantic: {aiContent?.semantic}</div>
|
||||
<form
|
||||
action={type === 'create' ? createPhotoAction : updatePhotoAction}
|
||||
onSubmit={() => blur()}
|
||||
@ -325,7 +174,7 @@ export default function PhotoForm({
|
||||
annotation: formatCount(count),
|
||||
annotationAria: formatCountDescriptive(count, 'tagged'),
|
||||
})),
|
||||
aiTextGeneration,
|
||||
aiContent !== undefined,
|
||||
)
|
||||
.map(([key, {
|
||||
label,
|
||||
|
||||
@ -72,12 +72,6 @@ const FORM_METADATA = (
|
||||
validateStringMaxLength: STRING_MAX_LENGTH_LONG,
|
||||
shouldHide: ({ title, caption }) => !title && !caption,
|
||||
},
|
||||
semanticDescription: {
|
||||
label: 'semantic description',
|
||||
capitalize: true,
|
||||
validateStringMaxLength: STRING_MAX_LENGTH_LONG,
|
||||
hide: !aiTextGeneration,
|
||||
},
|
||||
tags: {
|
||||
label: 'tags',
|
||||
tagOptions,
|
||||
@ -85,6 +79,12 @@ const FORM_METADATA = (
|
||||
? `'${TAG_FAVS}' is a reserved tag`
|
||||
: undefined,
|
||||
},
|
||||
semanticDescription: {
|
||||
label: 'semantic description',
|
||||
capitalize: true,
|
||||
validateStringMaxLength: STRING_MAX_LENGTH_LONG,
|
||||
hide: !aiTextGeneration,
|
||||
},
|
||||
id: { label: 'id', readOnly: true, hideIfEmpty: true },
|
||||
blurData: {
|
||||
label: 'blur data',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user