strtus/Vercel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Strip GPS data when uploading/syncing photos

Browse Source
This commit is contained in:
Sam Becker 2024-06-07 00:24:52 -05:00
parent 67c392bf62
commit 11362450f1
13 changed files with 145 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/admin/PhotoSyncButton.tsx
View File

@ -29,7 +29,7 @@ export default function PhotoSyncButton({
if (photoTitle) { confirmText.push(`"${photoTitle}"`); }
confirmText.push('data from original file?');
if (hasAiTextGeneration) { confirmText.push(
'This will also auto-generate AI text for undefined fields.'); }
'AI text will be generated for undefined fields.'); }
confirmText.push('This action cannot be undone.');
return (
<FormWithConfirm

2
src/app/admin/uploads/[uploadPath]/page.tsx
View File

@ -18,6 +18,7 @@ export default async function UploadPage({ params: { uploadPath } }: Params) {
blobId,
photoFormExif,
imageResizedBase64: imageThumbnailBase64,
shouldStripGpsData,
} = await extractImageDataFromBlobPath(uploadPath, {
includeInitialPhotoFields: true,
generateBlurData: BLUR_ENABLED,
@ -45,6 +46,7 @@ export default async function UploadPage({ params: { uploadPath } }: Params) {
hasAiTextGeneration,
textFieldsToAutoGenerate,
imageThumbnailBase64,
shouldStripGpsData,
}} />
);
};

3
src/photo/UploadPageClient.tsx
View File

@ -17,6 +17,7 @@ export default function UploadPageClient({
hasAiTextGeneration,
textFieldsToAutoGenerate,
imageThumbnailBase64,
shouldStripGpsData,
}: {
blobId?: string
photoFormExif: Partial<PhotoFormData>
@ -24,6 +25,7 @@ export default function UploadPageClient({
hasAiTextGeneration?: boolean
textFieldsToAutoGenerate?: AiAutoGeneratedField[],
imageThumbnailBase64?: string
shouldStripGpsData?: boolean
}) {
const {
pending,
@ -60,6 +62,7 @@ export default function UploadPageClient({
initialPhotoForm={initialPhotoForm}
uniqueTags={uniqueTags}
aiContent={hasAiTextGeneration ? aiContent : undefined}
shouldStripGpsData={shouldStripGpsData}
onTitleChange={setUpdatedTitle}
onTextContentChange={setHasTextContent}
onFormStatusChange={setIsPending}

30
src/photo/actions.ts
View File

@ -16,10 +16,7 @@ import {
convertPhotoToFormData,
} from './form';
import { redirect } from 'next/navigation';
import {
convertUploadToPhoto,
deleteStorageUrl,
} from '@/services/storage';
import { deleteFile } from '@/services/storage';
import {
getPhotosCached,
getPhotosMetaCached,
@ -49,6 +46,7 @@ import {
import { getStorageUploadUrlsNoStore } from '@/services/storage/cache';
import { generateAiImageQueries } from './ai/server';
import { createStreamableValue } from 'ai/rsc';
import { convertUploadToPhoto } from './storage';
// Private actions
@ -56,7 +54,10 @@ export const createPhotoAction = async (formData: FormData) =>
runAuthenticatedAdminServerAction(async () => {
const photo = convertFormDataToPhotoDbInsert(formData);
const updatedUrl = await convertUploadToPhoto(photo.url);
const updatedUrl = await convertUploadToPhoto(
photo.url,
formData.get('shouldStripGpsData') === 'true',
);
if (updatedUrl) {
photo.url = updatedUrl;
@ -103,6 +104,7 @@ export const addAllUploadsAction = async ({
const {
photoFormExif,
imageResizedBase64,
shouldStripGpsData,
} = await extractImageDataFromBlobPath(url, {
includeInitialPhotoFields: true,
generateBlurData: BLUR_ENABLED,
@ -144,7 +146,10 @@ export const addAllUploadsAction = async ({
addedUploadUrls: addedUploadUrls.join(','),
});
const updatedUrl = await convertUploadToPhoto(url);
const updatedUrl = await convertUploadToPhoto(
url,
shouldStripGpsData,
);
if (updatedUrl) {
stream.update({
headline,
@ -214,7 +219,7 @@ export const deletePhotoAction = async (
shouldRedirect?: boolean,
) =>
runAuthenticatedAdminServerAction(async () => {
await deletePhoto(photoId).then(() => deleteStorageUrl(photoUrl));
await deletePhoto(photoId).then(() => deleteFile(photoUrl));
revalidateAllKeysAndPaths();
if (shouldRedirect) {
redirect(PATH_ROOT);
@ -254,7 +259,7 @@ export const renamePhotoTagGloballyAction = async (formData: FormData) =>
export const deleteBlobPhotoAction = async (formData: FormData) =>
runAuthenticatedAdminServerAction(async () => {
await deleteStorageUrl(formData.get('url') as string);
await deleteFile(formData.get('url') as string);
revalidateAdminPaths();
@ -282,6 +287,7 @@ export const getExifDataAction = async (
// Accessed from admin photo table, will:
// - update EXIF data
// - anonymize storage url if necessary
// - strip GPS data if necessary
// - update blur data (or destroy if blur is disabled)
// - generate AI text data, if enabled, and auto-generated fields are empty
export const syncPhotoAction = async (formData: FormData) =>
@ -293,6 +299,7 @@ export const syncPhotoAction = async (formData: FormData) =>
const {
photoFormExif,
imageResizedBase64,
shouldStripGpsData,
} = await extractImageDataFromBlobPath(photo.url, {
includeInitialPhotoFields: false,
generateBlurData: BLUR_ENABLED,
@ -300,10 +307,13 @@ export const syncPhotoAction = async (formData: FormData) =>
});
if (photoFormExif) {
if (photo.url.includes(photo.id)) {
if (photo.url.includes(photo.id) || shouldStripGpsData) {
// Anonymize storage url on update if necessary by
// re-running image upload transfer logic
const url = await convertUploadToPhoto(photo.url);
const url = await convertUploadToPhoto(
photo.url,
shouldStripGpsData,
);
if (url) { photo.url = url; }
}

7
src/photo/form/PhotoForm.tsx
View File

@ -40,6 +40,7 @@ export default function PhotoForm({
updatedBlurData,
uniqueTags,
aiContent,
shouldStripGpsData,
onTitleChange,
onTextContentChange,
onFormStatusChange,
@ -50,6 +51,7 @@ export default function PhotoForm({
updatedBlurData?: string
uniqueTags?: TagsWithMeta
aiContent?: AiContent
shouldStripGpsData?: boolean
onTitleChange?: (updatedTitle: string) => void
onTextContentChange?: (hasContent: boolean) => void,
onFormStatusChange?: (pending: boolean) => void
@ -353,6 +355,11 @@ export default function PhotoForm({
type={type}
accessory={accessoryForField(key)}
/>)}
<input
type="hidden"
name="shouldStripGpsData"
value={shouldStripGpsData ? 'true' : 'false'}
/>
</div>
{/* Actions */}
<div className={clsx(

9
src/photo/server.ts
View File

@ -11,6 +11,7 @@ import { ExifData, ExifParserFactory } from 'ts-exif-parser';
import { PhotoFormData } from './form';
import { FilmSimulation } from '@/simulation';
import sharp, { Sharp } from 'sharp';
import { GEO_PRIVACY_ENABLED } from '@/site/config';
const IMAGE_WIDTH_RESIZE = 200;
const IMAGE_WIDTH_BLUR = 200;
@ -26,6 +27,7 @@ export const extractImageDataFromBlobPath = async (
blobId?: string
photoFormExif?: Partial<PhotoFormData>
imageResizedBase64?: string
shouldStripGpsData?: boolean
}> => {
const {
includeInitialPhotoFields,
@ -47,6 +49,7 @@ export const extractImageDataFromBlobPath = async (
let filmSimulation: FilmSimulation | undefined;
let blurData: string | undefined;
let imageResizedBase64: string | undefined;
let shouldStripGpsData = false;
if (fileBytes) {
const parser = ExifParserFactory.create(Buffer.from(fileBytes));
@ -74,6 +77,11 @@ export const extractImageDataFromBlobPath = async (
if (generateResizedImage) {
imageResizedBase64 = await resizeImage(fileBytes);
}
shouldStripGpsData = GEO_PRIVACY_ENABLED && (
Boolean(exifData.tags?.GPSLatitude) ||
Boolean(exifData.tags?.GPSLongitude)
);
}
return {
@ -91,6 +99,7 @@ export const extractImageDataFromBlobPath = async (
},
},
imageResizedBase64,
shouldStripGpsData,
};
};

28
src/photo/storage.ts Normal file
View File

@ -0,0 +1,28 @@
import {
deleteFile,
generateRandomFileNameForPhoto,
getExtensionFromStorageUrl,
moveFile,
putFile,
} from '@/services/storage';
import { stripGpsFromFile } from '@/utility/exif-server';
export const convertUploadToPhoto = async (
urlOrigin: string,
stripGps?: boolean,
) => {
const fileName = generateRandomFileNameForPhoto();
const fileExtension = getExtensionFromStorageUrl(urlOrigin);
const photoPath = `${fileName}.${fileExtension || 'jpg'}`;
if (stripGps) {
const fileBytes = await fetch(urlOrigin, { cache: 'no-store' })
.then(res => res.arrayBuffer());
const fileWithoutGps = await stripGpsFromFile(fileBytes);
return putFile(fileWithoutGps, photoPath).then(async url => {
if (url) { await deleteFile(urlOrigin); }
return url;
});
} else {
return moveFile(urlOrigin, photoPath);
}
};

12
src/services/storage/aws-s3.ts
View File

@ -32,6 +32,18 @@ export const isUrlFromAwsS3 = (url?: string) =>
export const awsS3PutObjectCommandForKey = (Key: string) =>
new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' });
export const awsS3Put = (
file: File | Blob,
fileName: string,
): Promise<string> =>
awsS3Client().send(new PutObjectCommand({
Bucket: AWS_S3_BUCKET,
Key: fileName,
Body: file,
ACL: 'public-read',
}))
.then(() => urlForKey(fileName));
export const awsS3Copy = async (
fileNameSource: string,
fileNameDestination: string,

11
src/services/storage/cloudflare-r2.ts
View File

@ -53,6 +53,17 @@ export const isUrlFromCloudflareR2 = (url?: string) => (
export const cloudflareR2PutObjectCommandForKey = (Key: string) =>
new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key });
export const cloudflareR2Put = (
file: File | Blob,
fileName: string,
): Promise<string> =>
cloudflareR2Client().send(new PutObjectCommand({
Bucket: CLOUDFLARE_R2_BUCKET,
Key: fileName,
Body: file,
}))
.then(() => urlForKey(fileName));
export const cloudflareR2Copy = async (
fileNameSource: string,
fileNameDestination: string,

78
src/services/storage/index.ts
View File

@ -3,6 +3,7 @@ import {
vercelBlobCopy,
vercelBlobDelete,
vercelBlobList,
vercelBlobPut,
vercelBlobUploadFromClient,
} from './vercel-blob';
import {
@ -10,6 +11,7 @@ import {
awsS3Copy,
awsS3Delete,
awsS3List,
awsS3Put,
isUrlFromAwsS3,
} from './aws-s3';
import {
@ -24,6 +26,7 @@ import {
cloudflareR2Copy,
cloudflareR2Delete,
cloudflareR2List,
cloudflareR2Put,
isUrlFromCloudflareR2,
} from './cloudflare-r2';
import { PATH_API_PRESIGNED_URL } from '@/site/paths';
@ -69,6 +72,9 @@ export const storageTypeFromUrl = (url: string): StorageType => {
const PREFIX_UPLOAD = 'upload';
const PREFIX_PHOTO = 'photo';
export const generateRandomFileNameForPhoto = () =>
`${PREFIX_PHOTO}-${generateStorageId()}`;
const REGEX_UPLOAD_PATH = new RegExp(
`(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
'i',
@ -129,67 +135,47 @@ export const uploadPhotoFromClient = async (
? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true)
: vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`);
const moveFile = async (
export const putFile = (
file: File | Blob,
fileName: string,
) => {
switch (CURRENT_STORAGE) {
case 'vercel-blob':
return vercelBlobPut(file, fileName);
case 'cloudflare-r2':
return cloudflareR2Put(file, fileName);
case 'aws-s3':
return awsS3Put(file, fileName);
}
};
export const copyFile = (
originUrl: string,
destinationFileName: string,
) => {
const storageType = storageTypeFromUrl(originUrl);
let url: string | undefined;
// Copy file
switch (storageType) {
): Promise<string> => {
switch (storageTypeFromUrl(originUrl)) {
case 'vercel-blob':
url = await vercelBlobCopy(
return vercelBlobCopy(
originUrl,
destinationFileName,
false,
);
break;
case 'cloudflare-r2':
url = await cloudflareR2Copy(
return cloudflareR2Copy(
getFileNameFromStorageUrl(originUrl),
destinationFileName,
false,
);
break;
case 'aws-s3':
url = await awsS3Copy(
return awsS3Copy(
originUrl,
destinationFileName,
false,
);
break;
}
// If successful, delete original file
if (url) {
switch (storageType) {
case 'vercel-blob':
await vercelBlobDelete(originUrl);
break;
case 'cloudflare-r2':
await cloudflareR2Delete(getFileNameFromStorageUrl(originUrl));
break;
case 'aws-s3':
await awsS3Delete(getFileNameFromStorageUrl(originUrl));
break;
}
}
return url;
};
export const convertUploadToPhoto = async (
urlOrigin: string,
) => {
const fileName = `${PREFIX_PHOTO}-${generateStorageId()}`;
const fileExtension = getExtensionFromStorageUrl(urlOrigin);
const photoPath = `${fileName}.${fileExtension || 'jpg'}`;
return moveFile(urlOrigin, photoPath);
};
export const deleteStorageUrl = (url: string) => {
export const deleteFile = (url: string) => {
switch (storageTypeFromUrl(url)) {
case 'vercel-blob':
return vercelBlobDelete(url);
@ -200,6 +186,16 @@ export const deleteStorageUrl = (url: string) => {
}
};
export const moveFile = async (
originUrl: string,
destinationFileName: string,
) => {
const url = await copyFile(originUrl, destinationFileName);
// If successful, delete original file
if (url) { await deleteFile(originUrl); }
return url;
};
const getStorageUrlsForPrefix = async (prefix = '') => {
const urls: StorageListResponse = [];

14
src/services/storage/vercel-blob.ts
View File

@ -1,5 +1,5 @@
import { PATH_API_VERCEL_BLOB_UPLOAD } from '@/site/paths';
import { copy, del, list } from '@vercel/blob';
import { copy, del, list, put } from '@vercel/blob';
import { upload } from '@vercel/blob/client';
const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
@ -17,7 +17,7 @@ export const isUrlFromVercelBlob = (url?: string) =>
export const vercelBlobUploadFromClient = async (
file: File | Blob,
fileName: string,
) =>
): Promise<string> =>
upload(
fileName,
file,
@ -28,6 +28,16 @@ export const vercelBlobUploadFromClient = async (
)
.then(({ url }) => url);
export const vercelBlobPut = (
file: File | Blob,
fileName: string,
): Promise<string> =>
put(fileName, file, {
addRandomSuffix: false,
access: 'public',
})
.then(({ url }) => url);
export const vercelBlobCopy = (
sourceUrl: string,
destinationFileName: string,

2
src/utility/exif-server.ts
View File

@ -1,6 +1,6 @@
import * as PiExif from 'piexifjs';
export const removeGpsFromFile = async (
export const stripGpsFromFile = async (
fileBytes: ArrayBuffer
): Promise<Blob> => {
const base64 = Buffer.from(fileBytes).toString('base64');

3
tsconfig.json
View File

@ -30,7 +30,8 @@
},
"include": [
"**/*.ts",
"**/*.tsx"
"**/*.tsx",
".next/types/**/*.ts"
],
"exclude": [
"node_modules"