From 11362450f10652db3733c294bfb02b5461a4d1c3 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Fri, 7 Jun 2024 00:24:52 -0500 Subject: [PATCH] Strip GPS data when uploading/syncing photos --- src/admin/PhotoSyncButton.tsx | 2 +- src/app/admin/uploads/[uploadPath]/page.tsx | 2 + src/photo/UploadPageClient.tsx | 3 + src/photo/actions.ts | 30 +++++--- src/photo/form/PhotoForm.tsx | 7 ++ src/photo/server.ts | 9 +++ src/photo/storage.ts | 28 ++++++++ src/services/storage/aws-s3.ts | 12 ++++ src/services/storage/cloudflare-r2.ts | 11 +++ src/services/storage/index.ts | 78 ++++++++++----------- src/services/storage/vercel-blob.ts | 14 +++- src/utility/exif-server.ts | 2 +- tsconfig.json | 3 +- 13 files changed, 145 insertions(+), 56 deletions(-) create mode 100644 src/photo/storage.ts diff --git a/src/admin/PhotoSyncButton.tsx b/src/admin/PhotoSyncButton.tsx index 61a1eb31..2eda5877 100644 --- a/src/admin/PhotoSyncButton.tsx +++ b/src/admin/PhotoSyncButton.tsx @@ -29,7 +29,7 @@ export default function PhotoSyncButton({ if (photoTitle) { confirmText.push(`"${photoTitle}"`); } confirmText.push('data from original file?'); if (hasAiTextGeneration) { confirmText.push( - 'This will also auto-generate AI text for undefined fields.'); } + 'AI text will be generated for undefined fields.'); } confirmText.push('This action cannot be undone.'); return ( ); }; diff --git a/src/photo/UploadPageClient.tsx b/src/photo/UploadPageClient.tsx index cbd106f1..ad579dd7 100644 --- a/src/photo/UploadPageClient.tsx +++ b/src/photo/UploadPageClient.tsx @@ -17,6 +17,7 @@ export default function UploadPageClient({ hasAiTextGeneration, textFieldsToAutoGenerate, imageThumbnailBase64, + shouldStripGpsData, }: { blobId?: string photoFormExif: Partial @@ -24,6 +25,7 @@ export default function UploadPageClient({ hasAiTextGeneration?: boolean textFieldsToAutoGenerate?: AiAutoGeneratedField[], imageThumbnailBase64?: string + shouldStripGpsData?: boolean }) { const { pending, @@ -60,6 +62,7 @@ export default function UploadPageClient({ initialPhotoForm={initialPhotoForm} uniqueTags={uniqueTags} aiContent={hasAiTextGeneration ? aiContent : undefined} + shouldStripGpsData={shouldStripGpsData} onTitleChange={setUpdatedTitle} onTextContentChange={setHasTextContent} onFormStatusChange={setIsPending} diff --git a/src/photo/actions.ts b/src/photo/actions.ts index e3b0ac4b..3878db82 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -16,10 +16,7 @@ import { convertPhotoToFormData, } from './form'; import { redirect } from 'next/navigation'; -import { - convertUploadToPhoto, - deleteStorageUrl, -} from '@/services/storage'; +import { deleteFile } from '@/services/storage'; import { getPhotosCached, getPhotosMetaCached, @@ -49,6 +46,7 @@ import { import { getStorageUploadUrlsNoStore } from '@/services/storage/cache'; import { generateAiImageQueries } from './ai/server'; import { createStreamableValue } from 'ai/rsc'; +import { convertUploadToPhoto } from './storage'; // Private actions @@ -56,7 +54,10 @@ export const createPhotoAction = async (formData: FormData) => runAuthenticatedAdminServerAction(async () => { const photo = convertFormDataToPhotoDbInsert(formData); - const updatedUrl = await convertUploadToPhoto(photo.url); + const updatedUrl = await convertUploadToPhoto( + photo.url, + formData.get('shouldStripGpsData') === 'true', + ); if (updatedUrl) { photo.url = updatedUrl; @@ -103,6 +104,7 @@ export const addAllUploadsAction = async ({ const { photoFormExif, imageResizedBase64, + shouldStripGpsData, } = await extractImageDataFromBlobPath(url, { includeInitialPhotoFields: true, generateBlurData: BLUR_ENABLED, @@ -144,7 +146,10 @@ export const addAllUploadsAction = async ({ addedUploadUrls: addedUploadUrls.join(','), }); - const updatedUrl = await convertUploadToPhoto(url); + const updatedUrl = await convertUploadToPhoto( + url, + shouldStripGpsData, + ); if (updatedUrl) { stream.update({ headline, @@ -214,7 +219,7 @@ export const deletePhotoAction = async ( shouldRedirect?: boolean, ) => runAuthenticatedAdminServerAction(async () => { - await deletePhoto(photoId).then(() => deleteStorageUrl(photoUrl)); + await deletePhoto(photoId).then(() => deleteFile(photoUrl)); revalidateAllKeysAndPaths(); if (shouldRedirect) { redirect(PATH_ROOT); @@ -254,7 +259,7 @@ export const renamePhotoTagGloballyAction = async (formData: FormData) => export const deleteBlobPhotoAction = async (formData: FormData) => runAuthenticatedAdminServerAction(async () => { - await deleteStorageUrl(formData.get('url') as string); + await deleteFile(formData.get('url') as string); revalidateAdminPaths(); @@ -282,6 +287,7 @@ export const getExifDataAction = async ( // Accessed from admin photo table, will: // - update EXIF data // - anonymize storage url if necessary +// - strip GPS data if necessary // - update blur data (or destroy if blur is disabled) // - generate AI text data, if enabled, and auto-generated fields are empty export const syncPhotoAction = async (formData: FormData) => @@ -293,6 +299,7 @@ export const syncPhotoAction = async (formData: FormData) => const { photoFormExif, imageResizedBase64, + shouldStripGpsData, } = await extractImageDataFromBlobPath(photo.url, { includeInitialPhotoFields: false, generateBlurData: BLUR_ENABLED, @@ -300,10 +307,13 @@ export const syncPhotoAction = async (formData: FormData) => }); if (photoFormExif) { - if (photo.url.includes(photo.id)) { + if (photo.url.includes(photo.id) || shouldStripGpsData) { // Anonymize storage url on update if necessary by // re-running image upload transfer logic - const url = await convertUploadToPhoto(photo.url); + const url = await convertUploadToPhoto( + photo.url, + shouldStripGpsData, + ); if (url) { photo.url = url; } } diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 8fdd37d9..867e9dfd 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -40,6 +40,7 @@ export default function PhotoForm({ updatedBlurData, uniqueTags, aiContent, + shouldStripGpsData, onTitleChange, onTextContentChange, onFormStatusChange, @@ -50,6 +51,7 @@ export default function PhotoForm({ updatedBlurData?: string uniqueTags?: TagsWithMeta aiContent?: AiContent + shouldStripGpsData?: boolean onTitleChange?: (updatedTitle: string) => void onTextContentChange?: (hasContent: boolean) => void, onFormStatusChange?: (pending: boolean) => void @@ -353,6 +355,11 @@ export default function PhotoForm({ type={type} accessory={accessoryForField(key)} />)} + {/* Actions */}
imageResizedBase64?: string + shouldStripGpsData?: boolean }> => { const { includeInitialPhotoFields, @@ -47,6 +49,7 @@ export const extractImageDataFromBlobPath = async ( let filmSimulation: FilmSimulation | undefined; let blurData: string | undefined; let imageResizedBase64: string | undefined; + let shouldStripGpsData = false; if (fileBytes) { const parser = ExifParserFactory.create(Buffer.from(fileBytes)); @@ -74,6 +77,11 @@ export const extractImageDataFromBlobPath = async ( if (generateResizedImage) { imageResizedBase64 = await resizeImage(fileBytes); } + + shouldStripGpsData = GEO_PRIVACY_ENABLED && ( + Boolean(exifData.tags?.GPSLatitude) || + Boolean(exifData.tags?.GPSLongitude) + ); } return { @@ -91,6 +99,7 @@ export const extractImageDataFromBlobPath = async ( }, }, imageResizedBase64, + shouldStripGpsData, }; }; diff --git a/src/photo/storage.ts b/src/photo/storage.ts new file mode 100644 index 00000000..ab3c3320 --- /dev/null +++ b/src/photo/storage.ts @@ -0,0 +1,28 @@ +import { + deleteFile, + generateRandomFileNameForPhoto, + getExtensionFromStorageUrl, + moveFile, + putFile, +} from '@/services/storage'; +import { stripGpsFromFile } from '@/utility/exif-server'; + +export const convertUploadToPhoto = async ( + urlOrigin: string, + stripGps?: boolean, +) => { + const fileName = generateRandomFileNameForPhoto(); + const fileExtension = getExtensionFromStorageUrl(urlOrigin); + const photoPath = `${fileName}.${fileExtension || 'jpg'}`; + if (stripGps) { + const fileBytes = await fetch(urlOrigin, { cache: 'no-store' }) + .then(res => res.arrayBuffer()); + const fileWithoutGps = await stripGpsFromFile(fileBytes); + return putFile(fileWithoutGps, photoPath).then(async url => { + if (url) { await deleteFile(urlOrigin); } + return url; + }); + } else { + return moveFile(urlOrigin, photoPath); + } +}; diff --git a/src/services/storage/aws-s3.ts b/src/services/storage/aws-s3.ts index ff93d725..1c7d3218 100644 --- a/src/services/storage/aws-s3.ts +++ b/src/services/storage/aws-s3.ts @@ -32,6 +32,18 @@ export const isUrlFromAwsS3 = (url?: string) => export const awsS3PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' }); +export const awsS3Put = ( + file: File | Blob, + fileName: string, +): Promise => + awsS3Client().send(new PutObjectCommand({ + Bucket: AWS_S3_BUCKET, + Key: fileName, + Body: file, + ACL: 'public-read', + })) + .then(() => urlForKey(fileName)); + export const awsS3Copy = async ( fileNameSource: string, fileNameDestination: string, diff --git a/src/services/storage/cloudflare-r2.ts b/src/services/storage/cloudflare-r2.ts index 9929b766..d556446c 100644 --- a/src/services/storage/cloudflare-r2.ts +++ b/src/services/storage/cloudflare-r2.ts @@ -53,6 +53,17 @@ export const isUrlFromCloudflareR2 = (url?: string) => ( export const cloudflareR2PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key }); +export const cloudflareR2Put = ( + file: File | Blob, + fileName: string, +): Promise => + cloudflareR2Client().send(new PutObjectCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + Key: fileName, + Body: file, + })) + .then(() => urlForKey(fileName)); + export const cloudflareR2Copy = async ( fileNameSource: string, fileNameDestination: string, diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 5e575ed9..e8c57e30 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -3,6 +3,7 @@ import { vercelBlobCopy, vercelBlobDelete, vercelBlobList, + vercelBlobPut, vercelBlobUploadFromClient, } from './vercel-blob'; import { @@ -10,6 +11,7 @@ import { awsS3Copy, awsS3Delete, awsS3List, + awsS3Put, isUrlFromAwsS3, } from './aws-s3'; import { @@ -24,6 +26,7 @@ import { cloudflareR2Copy, cloudflareR2Delete, cloudflareR2List, + cloudflareR2Put, isUrlFromCloudflareR2, } from './cloudflare-r2'; import { PATH_API_PRESIGNED_URL } from '@/site/paths'; @@ -69,6 +72,9 @@ export const storageTypeFromUrl = (url: string): StorageType => { const PREFIX_UPLOAD = 'upload'; const PREFIX_PHOTO = 'photo'; +export const generateRandomFileNameForPhoto = () => + `${PREFIX_PHOTO}-${generateStorageId()}`; + const REGEX_UPLOAD_PATH = new RegExp( `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`, 'i', @@ -129,67 +135,47 @@ export const uploadPhotoFromClient = async ( ? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true) : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`); -const moveFile = async ( +export const putFile = ( + file: File | Blob, + fileName: string, +) => { + switch (CURRENT_STORAGE) { + case 'vercel-blob': + return vercelBlobPut(file, fileName); + case 'cloudflare-r2': + return cloudflareR2Put(file, fileName); + case 'aws-s3': + return awsS3Put(file, fileName); + } +}; + +export const copyFile = ( originUrl: string, destinationFileName: string, -) => { - const storageType = storageTypeFromUrl(originUrl); - - let url: string | undefined; - - // Copy file - switch (storageType) { +): Promise => { + switch (storageTypeFromUrl(originUrl)) { case 'vercel-blob': - url = await vercelBlobCopy( + return vercelBlobCopy( originUrl, destinationFileName, false, ); - break; case 'cloudflare-r2': - url = await cloudflareR2Copy( + return cloudflareR2Copy( getFileNameFromStorageUrl(originUrl), destinationFileName, false, ); - break; case 'aws-s3': - url = await awsS3Copy( + return awsS3Copy( originUrl, destinationFileName, false, ); - break; } - - // If successful, delete original file - if (url) { - switch (storageType) { - case 'vercel-blob': - await vercelBlobDelete(originUrl); - break; - case 'cloudflare-r2': - await cloudflareR2Delete(getFileNameFromStorageUrl(originUrl)); - break; - case 'aws-s3': - await awsS3Delete(getFileNameFromStorageUrl(originUrl)); - break; - } - } - - return url; }; -export const convertUploadToPhoto = async ( - urlOrigin: string, -) => { - const fileName = `${PREFIX_PHOTO}-${generateStorageId()}`; - const fileExtension = getExtensionFromStorageUrl(urlOrigin); - const photoPath = `${fileName}.${fileExtension || 'jpg'}`; - return moveFile(urlOrigin, photoPath); -}; - -export const deleteStorageUrl = (url: string) => { +export const deleteFile = (url: string) => { switch (storageTypeFromUrl(url)) { case 'vercel-blob': return vercelBlobDelete(url); @@ -200,6 +186,16 @@ export const deleteStorageUrl = (url: string) => { } }; +export const moveFile = async ( + originUrl: string, + destinationFileName: string, +) => { + const url = await copyFile(originUrl, destinationFileName); + // If successful, delete original file + if (url) { await deleteFile(originUrl); } + return url; +}; + const getStorageUrlsForPrefix = async (prefix = '') => { const urls: StorageListResponse = []; diff --git a/src/services/storage/vercel-blob.ts b/src/services/storage/vercel-blob.ts index 293ab98a..8d6cc26c 100644 --- a/src/services/storage/vercel-blob.ts +++ b/src/services/storage/vercel-blob.ts @@ -1,5 +1,5 @@ import { PATH_API_VERCEL_BLOB_UPLOAD } from '@/site/paths'; -import { copy, del, list } from '@vercel/blob'; +import { copy, del, list, put } from '@vercel/blob'; import { upload } from '@vercel/blob/client'; const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match( @@ -17,7 +17,7 @@ export const isUrlFromVercelBlob = (url?: string) => export const vercelBlobUploadFromClient = async ( file: File | Blob, fileName: string, -) => +): Promise => upload( fileName, file, @@ -28,6 +28,16 @@ export const vercelBlobUploadFromClient = async ( ) .then(({ url }) => url); +export const vercelBlobPut = ( + file: File | Blob, + fileName: string, +): Promise => + put(fileName, file, { + addRandomSuffix: false, + access: 'public', + }) + .then(({ url }) => url); + export const vercelBlobCopy = ( sourceUrl: string, destinationFileName: string, diff --git a/src/utility/exif-server.ts b/src/utility/exif-server.ts index 668919b5..93c06f8e 100644 --- a/src/utility/exif-server.ts +++ b/src/utility/exif-server.ts @@ -1,6 +1,6 @@ import * as PiExif from 'piexifjs'; -export const removeGpsFromFile = async ( +export const stripGpsFromFile = async ( fileBytes: ArrayBuffer ): Promise => { const base64 = Buffer.from(fileBytes).toString('base64'); diff --git a/tsconfig.json b/tsconfig.json index 72d3c462..0bc1366b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,7 +30,8 @@ }, "include": [ "**/*.ts", - "**/*.tsx" + "**/*.tsx", + ".next/types/**/*.ts" ], "exclude": [ "node_modules"