strtus/Vercel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement upload -> photo copy in R2

Browse Source
This commit is contained in:
Sam Becker 2024-01-21 11:14:12 -06:00
parent 04dd1baef4
commit b4c0f24dde
9 changed files with 100 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/admin/BlobUrls.tsx → src/admin/StorageUrls.tsx
View File

@ -10,7 +10,7 @@ import { clsx } from 'clsx/lite';
import { pathForAdminUploadUrl } from '@/site/paths';
import AddButton from './AddButton';
export default function BlobUrls({
export default function StorageUrls({
title,
urls,
}: {

4
src/app/admin/layout.tsx
View File

@ -1,6 +1,6 @@
import AdminNav from '@/admin/AdminNav';
import {
getBlobUploadUrlsNoStore,
getStorageUploadUrlsNoStore,
getPhotosCountIncludingHiddenCached,
getUniqueTagsCached,
} from '@/cache';
@ -21,7 +21,7 @@ export default async function AdminLayout({
countTags,
] = await Promise.all([
getPhotosCountIncludingHiddenCached(),
getBlobUploadUrlsNoStore()
getStorageUploadUrlsNoStore()
.then(urls => urls.length)
.catch(e => {
console.error(`Error getting blob upload urls: ${e}`);

8
src/app/admin/photos/page.tsx
View File

@ -14,7 +14,7 @@ import {
import { titleForPhoto } from '@/photo';
import MorePhotos from '@/photo/MorePhotos';
import {
getBlobPhotoUrlsNoStore,
getStoragePhotoUrlsNoStore,
getPhotosCached,
getPhotosCountIncludingHiddenCached,
} from '@/cache';
@ -26,7 +26,7 @@ import {
import AdminGrid from '@/admin/AdminGrid';
import DeleteButton from '@/admin/DeleteButton';
import EditButton from '@/admin/EditButton';
import BlobUrls from '@/admin/BlobUrls';
import StorageUrls from '@/admin/StorageUrls';
import { PRO_MODE_ENABLED } from '@/site/config';
import SubmitButtonWithStatus from '@/components/SubmitButtonWithStatus';
import IconGrSync from '@/site/IconGrSync';
@ -45,7 +45,7 @@ export default async function AdminPhotosPage({
] = await Promise.all([
getPhotosCached({ includeHidden: true, sortBy: 'createdAt', limit }),
getPhotosCountIncludingHiddenCached(),
DEBUG_PHOTO_BLOBS ? getBlobPhotoUrlsNoStore() : [],
DEBUG_PHOTO_BLOBS ? getStoragePhotoUrlsNoStore() : [],
]);
const showMorePhotos = count > photos.length;
@ -60,7 +60,7 @@ export default async function AdminPhotosPage({
'border-b pb-6',
'border-gray-200 dark:border-gray-700',
)}>
<BlobUrls
<StorageUrls
title={`Photo Blobs (${blobPhotoUrls.length})`}
urls={blobPhotoUrls}
/>

8
src/app/admin/uploads/page.tsx
View File

@ -1,12 +1,12 @@
import BlobUrls from '@/admin/BlobUrls';
import { getBlobUploadUrlsNoStore } from '@/cache';
import StorageUrls from '@/admin/StorageUrls';
import { getStorageUploadUrlsNoStore } from '@/cache';
import SiteGrid from '@/components/SiteGrid';
export default async function AdminUploadsPage() {
const blobUrls = await getBlobUploadUrlsNoStore();
const storageUrls = await getStorageUploadUrlsNoStore();
return (
<SiteGrid
contentMain={<BlobUrls urls={blobUrls} />}
contentMain={<StorageUrls urls={storageUrls} />}
/>
);
}

20
src/cache/index.ts vendored
View File

@ -24,7 +24,7 @@ import {
getPhotosNearId,
} from '@/services/vercel-postgres';
import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo';
import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/storage';
import { getStoragePhotoUrls, getStorageUploadUrls } from '@/services/storage';
import type { Session } from 'next-auth';
import { createCameraKey } from '@/camera';
import { PATHS_ADMIN } from '@/site/paths';
@ -218,15 +218,17 @@ export const getPhotoNoStore = (...args: Parameters<typeof getPhoto>) => {
return getPhoto(...args);
};
export const getBlobUploadUrlsNoStore: typeof getBlobUploadUrls = (...args) => {
unstable_noStore();
return getBlobUploadUrls(...args);
};
export const getStorageUploadUrlsNoStore: typeof getStorageUploadUrls =
(...args) => {
unstable_noStore();
return getStorageUploadUrls(...args);
};
export const getBlobPhotoUrlsNoStore: typeof getBlobPhotoUrls = (...args) => {
unstable_noStore();
return getBlobPhotoUrls(...args);
};
export const getStoragePhotoUrlsNoStore: typeof getStoragePhotoUrls =
(...args) => {
unstable_noStore();
return getStoragePhotoUrls(...args);
};
export const getImageCacheHeadersForAuth = (session: Session | null) => {
return {

6
src/photo/actions.ts
View File

@ -16,7 +16,7 @@ import {
import { redirect } from 'next/navigation';
import {
convertUploadToPhoto,
deleteBlobUrl,
deleteStorageUrl,
} from '@/services/storage';
import {
revalidateAdminPaths,
@ -66,7 +66,7 @@ export async function toggleFavoritePhoto(photoId: string) {
export async function deletePhotoAction(formData: FormData) {
await Promise.all([
deleteBlobUrl(formData.get('url') as string),
deleteStorageUrl(formData.get('url') as string),
sqlDeletePhoto(formData.get('id') as string),
]);
@ -94,7 +94,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) {
}
export async function deleteBlobPhotoAction(formData: FormData) {
await deleteBlobUrl(formData.get('url') as string);
await deleteStorageUrl(formData.get('url') as string);
revalidateAdminPaths();

4
src/services/storage/aws-s3.ts
View File

@ -5,7 +5,7 @@ import {
ListObjectsCommand,
PutObjectCommand,
} from '@aws-sdk/client-s3';
import { generateBlobId } from '.';
import { generateStorageId } from '.';
const AWS_S3_BUCKET = process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? '';
const AWS_S3_REGION = process.env.NEXT_PUBLIC_AWS_S3_REGION ?? '';
@ -38,7 +38,7 @@ export const awsS3Copy = async (
const name = fileNameSource.split('.')[0];
const extension = fileNameSource.split('.')[1];
const Key = addRandomSuffix
? `${name}-${generateBlobId()}.${extension}`
? `${name}-${generateStorageId()}.${extension}`
: fileNameDestination;
return awsS3Client().send(new CopyObjectCommand({
Bucket: AWS_S3_BUCKET,

20
src/services/storage/cloudflare-r2.ts
View File

@ -3,7 +3,9 @@ import {
ListObjectsCommand,
PutObjectCommand,
DeleteObjectCommand,
CopyObjectCommand,
} from '@aws-sdk/client-s3';
import { generateStorageId } from '.';
const CLOUDFLARE_R2_BUCKET =
process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? '';
@ -44,6 +46,24 @@ export const isUrlFromCloudflareR2 = (url: string) =>
export const cloudflareR2PutObjectCommandForKey = (Key: string) =>
new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key });
export const cloudflareR2Copy = async (
fileNameSource: string,
fileNameDestination: string,
addRandomSuffix?: boolean,
) => {
const name = fileNameSource.split('.')[0];
const extension = fileNameSource.split('.')[1];
const Key = addRandomSuffix
? `${name}-${generateStorageId()}.${extension}`
: fileNameDestination;
return cloudflareR2Client().send(new CopyObjectCommand({
Bucket: CLOUDFLARE_R2_BUCKET,
CopySource: `${CLOUDFLARE_R2_BUCKET}/${fileNameSource}`,
Key,
}))
.then(() => urlForKey(fileNameDestination));
};
export const cloudflareR2List = async (Prefix: string) =>
cloudflareR2Client().send(new ListObjectsCommand({
Bucket: CLOUDFLARE_R2_BUCKET,

76
src/services/storage/index.ts
View File

@ -21,13 +21,14 @@ import {
import { generateNanoid } from '@/utility/nanoid';
import {
CLOUDFLARE_R2_BASE_URL_PUBLIC,
cloudflareR2Copy,
cloudflareR2Delete,
cloudflareR2List,
isUrlFromCloudflareR2,
} from './cloudflare-r2';
import { PATH_API_PRESIGNED_URL } from '@/site/paths';
export const generateBlobId = () => generateNanoid(16);
export const generateStorageId = () => generateNanoid(16);
export type StorageType =
'vercel-blob' |
@ -42,7 +43,7 @@ export const labelForStorage = (type: StorageType): string => {
}
};
export const blobBaseUrlForStorage = (type: StorageType) => {
export const baseUrlForStorage = (type: StorageType) => {
switch (type) {
case 'vercel-blob': return VERCEL_BLOB_BASE_URL;
case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC;
@ -93,7 +94,7 @@ export const getIdFromStorageUrl = (url: string) =>
export const isUploadPathnameValid = (pathname?: string) =>
pathname?.match(REGEX_UPLOAD_PATH);
const getFileNameFromBlobUrl = (url: string) =>
const getFileNameFromStorageUrl = (url: string) =>
(new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? '';
export const uploadFromClientViaPresignedUrl = async (
@ -103,14 +104,14 @@ export const uploadFromClientViaPresignedUrl = async (
addRandomSuffix?: boolean,
) => {
const key = addRandomSuffix
? `${fileName}-${generateBlobId()}.${extension}`
? `${fileName}-${generateStorageId()}.${extension}`
: `${fileName}.${extension}`;
const url = await fetch(`${PATH_API_PRESIGNED_URL}/${key}`)
.then((response) => response.text());
return fetch(url, { method: 'PUT', body: file })
.then(() => `${blobBaseUrlForStorage(STORAGE_PREFERENCE)}/${key}`);
.then(() => `${baseUrlForStorage(STORAGE_PREFERENCE)}/${key}`);
};
export const uploadPhotoFromClient = async (
@ -131,45 +132,74 @@ export const convertUploadToPhoto = async (
const fileExtension = getExtensionFromStorageUrl(uploadUrl);
const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
const useAwsS3 = HAS_AWS_S3_STORAGE && isUrlFromAwsS3(uploadUrl);
const storageType = storageTypeFromUrl(uploadUrl);
const url = await (useAwsS3
? awsS3Copy(uploadUrl, photoUrl, photoId === undefined)
: vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined));
let url: string | undefined;
// Copy file
switch (storageType) {
case 'vercel-blob':
url = await vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined);
break;
case 'cloudflare-r2':
url = await cloudflareR2Copy(
getFileNameFromStorageUrl(uploadUrl),
photoUrl,
photoId === undefined,
);
break;
case 'aws-s3':
url = await awsS3Copy(uploadUrl, photoUrl, photoId === undefined);
break;
}
// If successful, delete original file
if (url) {
await (useAwsS3
? awsS3Delete(getFileNameFromBlobUrl(uploadUrl))
: vercelBlobDelete(uploadUrl));
switch (storageType) {
case 'vercel-blob':
await vercelBlobDelete(uploadUrl);
break;
case 'cloudflare-r2':
await cloudflareR2Delete(getFileNameFromStorageUrl(uploadUrl));
break;
case 'aws-s3':
await awsS3Delete(getFileNameFromStorageUrl(uploadUrl));
break;
}
}
return url;
};
export const deleteBlobUrl = (url: string) => {
export const deleteStorageUrl = (url: string) => {
switch (storageTypeFromUrl(url)) {
case 'vercel-blob': return vercelBlobDelete(url);
case 'cloudflare-r2': return cloudflareR2Delete(getFileNameFromBlobUrl(url));
case 'aws-s3': return awsS3Delete(getFileNameFromBlobUrl(url));
case 'vercel-blob':
return vercelBlobDelete(url);
case 'cloudflare-r2':
return cloudflareR2Delete(getFileNameFromStorageUrl(url));
case 'aws-s3':
return awsS3Delete(getFileNameFromStorageUrl(url));
}
};
export const getBlobUploadUrls = async (): Promise<string[]> => {
const getStorageUrlsForPrefix = async (prefix = ''): Promise<string[]> => {
const urls: string[] = [];
if (HAS_VERCEL_BLOB_STORAGE) {
urls.push(...await vercelBlobList(`${PREFIX_UPLOAD}-`));
urls.push(...await vercelBlobList(prefix));
}
if (HAS_AWS_S3_STORAGE) {
urls.push(...await awsS3List(`${PREFIX_UPLOAD}-`));
urls.push(...await awsS3List(prefix));
}
if (HAS_CLOUDFLARE_R2_STORAGE) {
urls.push(...await cloudflareR2List(`${PREFIX_UPLOAD}-`));
urls.push(...await cloudflareR2List(prefix));
}
return urls;
};
export const getBlobPhotoUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
? awsS3List(`${PREFIX_PHOTO}-`)
: vercelBlobList(`${PREFIX_PHOTO}-`);
export const getStorageUploadUrls = () =>
getStorageUrlsForPrefix(`${PREFIX_UPLOAD}-`);
export const getStoragePhotoUrls = () =>
getStorageUrlsForPrefix(`${PREFIX_PHOTO}-`);