strtus/Vercel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #190 from sambecker/ai-tag-refinement

Browse Source 
Refine AI text generation
This commit is contained in:
Sam Becker 2025-02-05 23:39:46 -06:00 committed by GitHub
commit 166a459593
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 87 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -77,11 +77,11 @@ _⚠️ READ BEFORE PROCEEDING_
3. Configure auto-generated fields (optional)
- Set which text fields auto-generate when uploading a photo by storing a comma-separated list, e.g., `AI_TEXT_AUTO_GENERATED_FIELDS = title, semantic`
- Accepted values:
- `all` (default)
- `title`
- `all`
- `title` (default)
- `caption`
- `tags`
- `semantic`
- `tags` (default)
- `semantic` (default)
- `none`
### Web Analytics

35
__tests__/ai.test.ts
View File

@ -1,68 +1,69 @@
/* eslint-disable quotes */
import {
parseAiAutoGeneratedFieldsText,
AI_AUTO_GENERATED_FIELDS_DEFAULT,
parseAiAutoGeneratedFieldsString,
parseTitleAndCaption,
} from "@/photo/ai";
describe('AI parses', () => {
describe('auto-generated fields', () => {
it('with spaces', () => {
expect(parseAiAutoGeneratedFieldsText())
expect(parseAiAutoGeneratedFieldsString())
.toStrictEqual(AI_AUTO_GENERATED_FIELDS_DEFAULT);
expect(parseAiAutoGeneratedFieldsString('all'))
.toStrictEqual(['title', 'caption', 'tags', 'semantic']);
expect(parseAiAutoGeneratedFieldsText('all'))
.toStrictEqual(['title', 'caption', 'tags', 'semantic']);
expect(parseAiAutoGeneratedFieldsText('title'))
expect(parseAiAutoGeneratedFieldsString('title'))
.toStrictEqual(['title']);
expect(parseAiAutoGeneratedFieldsText('title, caption'))
expect(parseAiAutoGeneratedFieldsString('title, caption'))
.toStrictEqual(['title', 'caption']);
expect(parseAiAutoGeneratedFieldsText('title, caption, invalid'))
expect(parseAiAutoGeneratedFieldsString('title, caption, invalid'))
.toStrictEqual(['title', 'caption']);
expect(parseAiAutoGeneratedFieldsText('title, caption, invalid, tags'))
expect(parseAiAutoGeneratedFieldsString('title, caption, invalid, tags'))
.toStrictEqual(['title', 'caption', 'tags']);
expect(parseAiAutoGeneratedFieldsText('none'))
expect(parseAiAutoGeneratedFieldsString('none'))
.toStrictEqual([]);
});
it('without spaces', () => {
expect(parseAiAutoGeneratedFieldsText('title,caption'))
expect(parseAiAutoGeneratedFieldsString('title,caption'))
.toStrictEqual(['title', 'caption']);
expect(parseAiAutoGeneratedFieldsText('title,caption,invalid'))
expect(parseAiAutoGeneratedFieldsString('title,caption,invalid'))
.toStrictEqual(['title', 'caption']);
expect(parseAiAutoGeneratedFieldsText('title,caption,invalid,tags'))
expect(parseAiAutoGeneratedFieldsString('title,caption,invalid,tags'))
.toStrictEqual(['title', 'caption', 'tags']);
});
});
it('received titles and captions', () => {
// Complex case
expect(parseTitleAndCaption(
`'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'`
`'Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."'`,
)).toStrictEqual({
title: 'Ephemeral Beauty',
caption: 'Roses bask in fleeting sunlight',
});
// Without surrounding single quotes
expect(parseTitleAndCaption(
`Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."`
`Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight."`,
)).toStrictEqual({
title: 'Ephemeral Beauty',
caption: 'Roses bask in fleeting sunlight',
});
// Without trailing period
expect(parseTitleAndCaption(
`Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight"`
`Title: "Ephemeral Beauty" Caption: "Roses bask in fleeting sunlight"`,
)).toStrictEqual({
title: 'Ephemeral Beauty',
caption: 'Roses bask in fleeting sunlight',
});
// Without and quotes
expect(parseTitleAndCaption(
`Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`
`Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`,
)).toStrictEqual({
title: 'Ephemeral Beauty',
caption: 'Roses bask in fleeting sunlight',
});
// With single space
expect(parseTitleAndCaption(
`Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`
`Title: Ephemeral Beauty Caption: Roses bask in fleeting sunlight`,
)).toStrictEqual({
title: 'Ephemeral Beauty',
caption: 'Roses bask in fleeting sunlight',

3
src/photo/PhotoLarge.tsx
View File

@ -265,7 +265,8 @@ export default function PhotoLarge({
<span
className={clsx(
'text-extra-dim',
'hover:underline decoration-dotted',
'decoration-dotted underline-offset-[3px]',
'hover:underline',
)}
>
{photo.focalLengthIn35MmFormatFormatted}

12
src/photo/actions.ts
View File

@ -9,6 +9,7 @@ import {
getPhoto,
getPhotos,
addTagsToPhotos,
getUniqueTags,
} from '@/photo/db/query';
import { GetPhotosOptions, areOptionsSensitive } from './db';
import {
@ -37,7 +38,7 @@ import { blurImageFromUrl, extractImageDataFromBlobPath } from './server';
import { TAG_FAVS, isTagFavs } from '@/tag';
import { convertPhotoToPhotoDbInsert, Photo } from '.';
import { runAuthenticatedAdminServerAction } from '@/auth';
import { AI_IMAGE_QUERIES, AiImageQuery } from './ai';
import { AiImageQuery, getAiImageQuery } from './ai';
import { streamOpenAiImageQuery } from '@/services/openai';
import {
AI_TEXT_AUTO_GENERATED_FIELDS,
@ -394,8 +395,13 @@ export const streamAiImageQueryAction = async (
imageBase64: string,
query: AiImageQuery,
) =>
runAuthenticatedAdminServerAction(() =>
streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query]));
runAuthenticatedAdminServerAction(async () => {
const existingTags = await getUniqueTags();
return streamOpenAiImageQuery(
imageBase64,
getAiImageQuery(query, existingTags),
);
});
export const getImageBlurAction = async (url: string) =>
runAuthenticatedAdminServerAction(() => blurImageFromUrl(url));

4
src/photo/ai/AiButton.tsx
View File

@ -1,12 +1,12 @@
import { AiContent } from './useAiImageQueries';
import { HiSparkles } from 'react-icons/hi';
import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.';
import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.';
import { useMemo } from 'react';
import LoaderButton from '@/components/primitives/LoaderButton';
export default function AiButton({
aiContent,
requestFields = ALL_AI_AUTO_GENERATED_FIELDS,
requestFields = AI_AUTO_GENERATED_FIELDS_ALL,
shouldConfirm,
className,
}: {

46
src/photo/ai/index.ts
View File

@ -1,32 +1,40 @@
/* eslint-disable max-len */
import { Tags } from '@/tag';
export type AiAutoGeneratedField =
'title' |
'caption' |
'tags' |
'semantic'
export const ALL_AI_AUTO_GENERATED_FIELDS: AiAutoGeneratedField[] = [
export const AI_AUTO_GENERATED_FIELDS_ALL: AiAutoGeneratedField[] = [
'title',
'caption',
'tags',
'semantic',
];
export const parseAiAutoGeneratedFieldsText = (
text = 'all',
export const AI_AUTO_GENERATED_FIELDS_DEFAULT: AiAutoGeneratedField[] = [
'title',
'tags',
'semantic',
];
export const parseAiAutoGeneratedFieldsString = (
text = AI_AUTO_GENERATED_FIELDS_DEFAULT.join(','),
): AiAutoGeneratedField[] => {
const textFormatted = text.trim().toLocaleLowerCase();
if (textFormatted === 'none') {
return [];
} else if (textFormatted === 'all') {
return ALL_AI_AUTO_GENERATED_FIELDS;
return AI_AUTO_GENERATED_FIELDS_ALL;
} else {
const fields = textFormatted
.toLocaleLowerCase()
.split(',')
.map(field => field.trim())
.filter(field => ALL_AI_AUTO_GENERATED_FIELDS
.filter(field => AI_AUTO_GENERATED_FIELDS_ALL
.includes(field as AiAutoGeneratedField));
return fields as AiAutoGeneratedField[];
}
@ -42,15 +50,25 @@ export type AiImageQuery =
'description-large' |
'description-semantic';
export const AI_IMAGE_QUERIES: Record<AiImageQuery, string> = {
'title': 'Write a compelling title for this image in 3 words or less',
'caption': 'Write a pithy caption for this image in 6 words or less and no punctuation',
'title-and-caption': 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"',
'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs',
'description-small': 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"',
'description': 'Describe this image',
'description-large': 'Describe this image in detail',
'description-semantic': 'List up to 5 things in this image without description as a comma-separated list',
export const getAiImageQuery = (
query: AiImageQuery,
existingTags: Tags = [],
): string => {
switch (query) {
case 'title': return 'Write a compelling title for this image in 3 words or less';
case 'caption': return 'Write a pithy caption for this image in 6 words or less and no punctuation';
case 'title-and-caption': return 'Write a compelling title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"';
case 'tags':
const tagQuery = 'Describe this image in 1-2 comma-separated unique keywords, with no adjective or adverbs. Avoid using general terms like "nature," "travel," "architecture," or "sky." Use terms that are highly specific to the image and not redundant.';
const tags = existingTags.map(({ tag }) => tag).join(', ');
return tags
? `${tagQuery}. Consider using some of these existing tags, but only if they are relevant: ${tags}.`
: tagQuery;
case 'description-small': return 'Describe this image succinctly without the initial text "This image shows" or "This is a picture of"';
case 'description': return 'Describe this image';
case 'description-large': return 'Describe this image in detail';
case 'description-semantic': return 'List up to 5 things in this image without description as a comma-separated list';
}
};
export const parseTitleAndCaption = (text: string) => {

14
src/photo/ai/server.ts
View File

@ -1,9 +1,10 @@
import { generateOpenAiImageQuery } from '@/services/openai';
import {
AI_IMAGE_QUERIES,
AiAutoGeneratedField,
getAiImageQuery,
parseTitleAndCaption,
} from '.';
import { getUniqueTags } from '../db/query';
export const generateAiImageQueries = async (
imageBase64?: string,
@ -29,7 +30,7 @@ export const generateAiImageQueries = async (
) {
const titleAndCaption = await generateOpenAiImageQuery(
imageBase64,
AI_IMAGE_QUERIES['title-and-caption'],
getAiImageQuery('title-and-caption'),
);
if (titleAndCaption) {
const titleAndCaptionParsed = parseTitleAndCaption(titleAndCaption);
@ -40,28 +41,29 @@ export const generateAiImageQueries = async (
if (textFieldsToGenerate.includes('title')) {
title = await generateOpenAiImageQuery(
imageBase64,
AI_IMAGE_QUERIES['title'],
getAiImageQuery('title'),
);
}
if (textFieldsToGenerate.includes('caption')) {
caption = await generateOpenAiImageQuery(
imageBase64,
AI_IMAGE_QUERIES['caption'],
getAiImageQuery('caption'),
);
}
}
if (textFieldsToGenerate.includes('tags')) {
const existingTags = await getUniqueTags();
tags = await generateOpenAiImageQuery(
imageBase64,
AI_IMAGE_QUERIES['tags'],
getAiImageQuery('tags', existingTags),
);
}
if (textFieldsToGenerate.includes('semantic')) {
semanticDescription = await generateOpenAiImageQuery(
imageBase64,
AI_IMAGE_QUERIES['description-small'],
getAiImageQuery('description-small'),
);
}
}

4
src/photo/ai/useAiImageQueries.ts
View File

@ -1,7 +1,7 @@
import { useCallback, useEffect, useRef } from 'react';
import useAiImageQuery from './useAiImageQuery';
import useTitleCaptionAiImageQuery from './useTitleCaptionAiImageQuery';
import { ALL_AI_AUTO_GENERATED_FIELDS, AiAutoGeneratedField } from '.';
import { AI_AUTO_GENERATED_FIELDS_ALL, AiAutoGeneratedField } from '.';
export type AiContent = ReturnType<typeof useAiImageQueries>;
@ -59,7 +59,7 @@ export default function useAiImageQueries(
const hasRunAllQueriesOnce = useRef(false);
const request = useCallback(async (
fields = ALL_AI_AUTO_GENERATED_FIELDS,
fields = AI_AUTO_GENERATED_FIELDS_ALL,
) => {
if (process.env.NODE_ENV !== 'production') {
console.log('RUNNING AI QUERIES', fields);

2
src/services/openai.ts
View File

@ -73,7 +73,7 @@ export const streamOpenAiImageQuery = async (
if (args) {
(async () => {
const { textStream } = await streamText(args);
const { textStream } = streamText(args);
for await (const delta of textStream) {
stream.update(cleanUpAiTextResponse(delta));
}

4
src/site/SiteChecklistClient.tsx
View File

@ -424,7 +424,9 @@ export default function SiteChecklistClient({
>
Comma-separated fields to auto-generate when
uploading photos. Accepted values: title, caption,
tags, description, all, or none (default is {'"all"'}):
tags, description, all, or none
{' '}
(default: {'"title, tags, semantic"'}):
{renderEnvVars(['AI_TEXT_AUTO_GENERATED_FIELDS'])}
</ChecklistRow>
</Checklist>

9
src/site/config.ts
View File

@ -1,4 +1,7 @@
import { parseAiAutoGeneratedFieldsText } from '@/photo/ai';
import {
AI_AUTO_GENERATED_FIELDS_DEFAULT,
parseAiAutoGeneratedFieldsString,
} from '@/photo/ai';
import type { StorageType } from '@/services/storage';
import { makeUrlAbsolute, shortenUrl } from '@/utility/url';
@ -142,7 +145,7 @@ export const CURRENT_STORAGE: StorageType =
export const AI_TEXT_GENERATION_ENABLED =
Boolean(process.env.OPENAI_SECRET_KEY);
export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsText(
export const AI_TEXT_AUTO_GENERATED_FIELDS = parseAiAutoGeneratedFieldsString(
process.env.AI_TEXT_AUTO_GENERATED_FIELDS);
// PERFORMANCE
@ -265,7 +268,7 @@ export const CONFIG_CHECKLIST_STATUS = {
? AI_TEXT_AUTO_GENERATED_FIELDS.length === 0
? ['none']
: AI_TEXT_AUTO_GENERATED_FIELDS
: ['all'],
: AI_AUTO_GENERATED_FIELDS_DEFAULT,
hasAiTextAutoGeneratedFields:
Boolean(process.env.AI_TEXT_AUTO_GENERATED_FIELDS),
// Performance