diff --git a/.vscode/settings.json b/.vscode/settings.json index e664ce2a..84c45555 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "ARROWRIGHT", "Astia", "camelcase", + "cloudflarestorage", "CredentialsSignin", "Eterna", "exif", diff --git a/README.md b/README.md index 919f2553..d11f01b8 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,42 @@ Installation - `NEXT_PUBLIC_GRID_ASPECT_RATIO = 1.5` sets aspect ratio for grid tiles (defaults to `1`—setting to `0` removes the constraint) - `NEXT_PUBLIC_OG_TEXT_ALIGNMENT = BOTTOM` keeps OG image text bottom aligned (default is top) -### Setup alternate storage +## Configure alternate storage -#### AWS S3 +Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations). If you have multiple adapters, you can set one as preferred by storing "aws-s3," "cloudflare-r2," or "vercel-blob" in `NEXT_PUBLIC_STORAGE_PREFERENCE`. + +### Cloudflare R2 + +1. Setup bucket + - [Create R2 bucket](https://developers.cloudflare.com/r2/) with default settings + - Setup CORS under bucket settings: + ```json + [{ + "AllowedHeaders": ["*"] + "AllowedOrigins": [ + "http://localhost:3000", + "https://{VERCEL_PROJECT_NAME}*.vercel.app", + "{PRODUCTION_DOMAIN}" + ], + "AllowedMethods": [ + "GET", + "PUT" + ], + }] + ``` + - Enable R2.dev subdomain (necessary in order to serve files publicly without a custom domain) + - Store configuration: + - `NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET`: bucket name + - `NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID`: account id (found on R2 overview page) + - `NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN`: r2.dev subdomain, e.g., "pub-jf90908..." +2. Setup credentials + - Create API token by selecting "Manage R2 API Tokens," and clicking "Create API Token" + - Select "Object Read & Write," choose "Apply to specific buckets only," and select the bucket created in Step 1. + - Store credentials (⚠️ _Ensure access keys are not prefixed with `NEXT_PUBLIC`_): + - `CLOUDFLARE_R2_ACCESS_KEY` + - `CLOUDFLARE_R2_SECRET_ACCESS_KEY` + +### AWS S3 1. Setup bucket - [Create S3 bucket](https://s3.console.aws.amazon.com/s3) with "ACLs enabled," and "Block all public access" turned off diff --git a/next.config.js b/next.config.js index 9ee18918..bd880f53 100644 --- a/next.config.js +++ b/next.config.js @@ -6,6 +6,11 @@ const VERCEL_BLOB_HOSTNAME = VERCEL_BLOB_STORE_ID ? `${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com` : undefined; +const CLOUDFLARE_R2_HOSTNAME = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN + ? `${process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev` + : undefined; + const AWS_S3_HOSTNAME = process.env.NEXT_PUBLIC_AWS_S3_BUCKET && process.env.NEXT_PUBLIC_AWS_S3_REGION @@ -28,6 +33,7 @@ const nextConfig = { imageSizes: [200], remotePatterns: [] .concat(createRemotePattern(VERCEL_BLOB_HOSTNAME)) + .concat(createRemotePattern(CLOUDFLARE_R2_HOSTNAME)) .concat(createRemotePattern(AWS_S3_HOSTNAME)), minimumCacheTTL: 31536000, }, diff --git a/src/admin/BlobUrls.tsx b/src/admin/StorageUrls.tsx similarity index 92% rename from src/admin/BlobUrls.tsx rename to src/admin/StorageUrls.tsx index 29020f8e..c4c1ef55 100644 --- a/src/admin/BlobUrls.tsx +++ b/src/admin/StorageUrls.tsx @@ -2,7 +2,7 @@ import { Fragment } from 'react'; import AdminGrid from './AdminGrid'; import Link from 'next/link'; import ImageTiny from '@/components/ImageTiny'; -import { fileNameForBlobUrl } from '@/services/blob'; +import { fileNameForStorageUrl } from '@/services/storage'; import FormWithConfirm from '@/components/FormWithConfirm'; import { deleteBlobPhotoAction } from '@/photo/actions'; import DeleteButton from './DeleteButton'; @@ -10,7 +10,7 @@ import { clsx } from 'clsx/lite'; import { pathForAdminUploadUrl } from '@/site/paths'; import AddButton from './AddButton'; -export default function BlobUrls({ +export default function StorageUrls({ title, urls, }: { @@ -21,7 +21,7 @@ export default function BlobUrls({ {urls.map(url => { const addUploadPath = pathForAdminUploadUrl(url); - const uploadFileName = fileNameForBlobUrl(url); + const uploadFileName = fileNameForStorageUrl(url); return urls.length) .catch(e => { console.error(`Error getting blob upload urls: ${e}`); diff --git a/src/app/admin/photos/page.tsx b/src/app/admin/photos/page.tsx index 2474f3d2..27f53955 100644 --- a/src/app/admin/photos/page.tsx +++ b/src/app/admin/photos/page.tsx @@ -14,7 +14,7 @@ import { import { titleForPhoto } from '@/photo'; import MorePhotos from '@/photo/MorePhotos'; import { - getBlobPhotoUrlsNoStore, + getStoragePhotoUrlsNoStore, getPhotosCached, getPhotosCountIncludingHiddenCached, } from '@/cache'; @@ -26,7 +26,7 @@ import { import AdminGrid from '@/admin/AdminGrid'; import DeleteButton from '@/admin/DeleteButton'; import EditButton from '@/admin/EditButton'; -import BlobUrls from '@/admin/BlobUrls'; +import StorageUrls from '@/admin/StorageUrls'; import { PRO_MODE_ENABLED } from '@/site/config'; import SubmitButtonWithStatus from '@/components/SubmitButtonWithStatus'; import IconGrSync from '@/site/IconGrSync'; @@ -45,7 +45,7 @@ export default async function AdminPhotosPage({ ] = await Promise.all([ getPhotosCached({ includeHidden: true, sortBy: 'createdAt', limit }), getPhotosCountIncludingHiddenCached(), - DEBUG_PHOTO_BLOBS ? getBlobPhotoUrlsNoStore() : [], + DEBUG_PHOTO_BLOBS ? getStoragePhotoUrlsNoStore() : [], ]); const showMorePhotos = count > photos.length; @@ -60,7 +60,7 @@ export default async function AdminPhotosPage({ 'border-b pb-6', 'border-gray-200 dark:border-gray-700', )}> - diff --git a/src/app/admin/uploads/page.tsx b/src/app/admin/uploads/page.tsx index 9e4e281a..8108e6eb 100644 --- a/src/app/admin/uploads/page.tsx +++ b/src/app/admin/uploads/page.tsx @@ -1,12 +1,12 @@ -import BlobUrls from '@/admin/BlobUrls'; -import { getBlobUploadUrlsNoStore } from '@/cache'; +import StorageUrls from '@/admin/StorageUrls'; +import { getStorageUploadUrlsNoStore } from '@/cache'; import SiteGrid from '@/components/SiteGrid'; export default async function AdminUploadsPage() { - const blobUrls = await getBlobUploadUrlsNoStore(); + const storageUrls = await getStorageUploadUrlsNoStore(); return ( } + contentMain={} /> ); } diff --git a/src/app/api/aws-s3/presigned-url/[key]/route.ts b/src/app/api/storage/presigned-url/[key]/route.ts similarity index 57% rename from src/app/api/aws-s3/presigned-url/[key]/route.ts rename to src/app/api/storage/presigned-url/[key]/route.ts index e2e858fe..da506a22 100644 --- a/src/app/api/aws-s3/presigned-url/[key]/route.ts +++ b/src/app/api/storage/presigned-url/[key]/route.ts @@ -2,7 +2,12 @@ import { auth } from '@/auth'; import { awsS3Client, awsS3PutObjectCommandForKey, -} from '@/services/blob/aws-s3'; +} from '@/services/storage/aws-s3'; +import { + cloudflareR2Client, + cloudflareR2PutObjectCommandForKey, +} from '@/services/storage/cloudflare-r2'; +import { CURRENT_STORAGE } from '@/site/config'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const runtime = 'edge'; @@ -14,8 +19,12 @@ export async function GET( const session = await auth(); if (session?.user && key) { const url = await getSignedUrl( - awsS3Client(), - awsS3PutObjectCommandForKey(key), + CURRENT_STORAGE === 'cloudflare-r2' + ? cloudflareR2Client() + : awsS3Client(), + CURRENT_STORAGE === 'cloudflare-r2' + ? cloudflareR2PutObjectCommandForKey(key) + : awsS3PutObjectCommandForKey(key), { expiresIn: 3600 } ); return new Response( diff --git a/src/app/admin/uploads/blob/route.tsx b/src/app/api/storage/vercel-blob/route.ts similarity index 92% rename from src/app/admin/uploads/blob/route.tsx rename to src/app/api/storage/vercel-blob/route.ts index 22b14c3d..e41543c6 100644 --- a/src/app/admin/uploads/blob/route.tsx +++ b/src/app/api/storage/vercel-blob/route.ts @@ -4,12 +4,12 @@ import { ACCEPTED_PHOTO_FILE_TYPES, MAX_PHOTO_UPLOAD_SIZE_IN_BYTES, } from '@/photo'; -import { isUploadPathnameValid } from '@/services/blob'; +import { isUploadPathnameValid } from '@/services/storage'; import { handleUpload, type HandleUploadBody } from '@vercel/blob/client'; import { NextResponse } from 'next/server'; export async function POST(request: Request): Promise { - const body = (await request.json()) as HandleUploadBody; + const body: HandleUploadBody = await request.json(); try { const jsonResponse = await handleUpload({ diff --git a/src/cache/index.ts b/src/cache/index.ts index ce35f63b..179bbbbb 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -24,7 +24,7 @@ import { getPhotosNearId, } from '@/services/vercel-postgres'; import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo'; -import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob'; +import { getStoragePhotoUrls, getStorageUploadUrls } from '@/services/storage'; import type { Session } from 'next-auth'; import { createCameraKey } from '@/camera'; import { PATHS_ADMIN } from '@/site/paths'; @@ -218,15 +218,17 @@ export const getPhotoNoStore = (...args: Parameters) => { return getPhoto(...args); }; -export const getBlobUploadUrlsNoStore: typeof getBlobUploadUrls = (...args) => { - unstable_noStore(); - return getBlobUploadUrls(...args); -}; +export const getStorageUploadUrlsNoStore: typeof getStorageUploadUrls = + (...args) => { + unstable_noStore(); + return getStorageUploadUrls(...args); + }; -export const getBlobPhotoUrlsNoStore: typeof getBlobPhotoUrls = (...args) => { - unstable_noStore(); - return getBlobPhotoUrls(...args); -}; +export const getStoragePhotoUrlsNoStore: typeof getStoragePhotoUrls = + (...args) => { + unstable_noStore(); + return getStoragePhotoUrls(...args); + }; export const getImageCacheHeadersForAuth = (session: Session | null) => { return { diff --git a/src/photo/PhotoUpload.tsx b/src/photo/PhotoUpload.tsx index 71d6a12f..b50fcdff 100644 --- a/src/photo/PhotoUpload.tsx +++ b/src/photo/PhotoUpload.tsx @@ -1,7 +1,7 @@ 'use client'; import { useState } from 'react'; -import { uploadPhotoFromClient } from '@/services/blob'; +import { uploadPhotoFromClient } from '@/services/storage'; import { useRouter } from 'next/navigation'; import { PATH_ADMIN_UPLOADS, pathForAdminUploadUrl } from '@/site/paths'; import ImageInput from '../components/ImageInput'; diff --git a/src/photo/actions.ts b/src/photo/actions.ts index f017d0f4..bbbe1865 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -16,8 +16,8 @@ import { import { redirect } from 'next/navigation'; import { convertUploadToPhoto, - deleteBlobUrl, -} from '@/services/blob'; + deleteStorageUrl, +} from '@/services/storage'; import { revalidateAdminPaths, revalidateAllKeysAndPaths, @@ -66,7 +66,7 @@ export async function toggleFavoritePhoto(photoId: string) { export async function deletePhotoAction(formData: FormData) { await Promise.all([ - deleteBlobUrl(formData.get('url') as string), + deleteStorageUrl(formData.get('url') as string), sqlDeletePhoto(formData.get('id') as string), ]); @@ -94,7 +94,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) { } export async function deleteBlobPhotoAction(formData: FormData) { - await deleteBlobUrl(formData.get('url') as string); + await deleteStorageUrl(formData.get('url') as string); revalidateAdminPaths(); diff --git a/src/photo/server.ts b/src/photo/server.ts index 7971c917..ce40fe79 100644 --- a/src/photo/server.ts +++ b/src/photo/server.ts @@ -1,7 +1,7 @@ import { - getExtensionFromBlobUrl, - getIdFromBlobUrl, -} from '@/services/blob'; + getExtensionFromStorageUrl, + getIdFromStorageUrl, +} from '@/services/storage'; import { convertExifToFormData } from '@/photo/form'; import { getFujifilmSimulationFromMakerNote, @@ -19,9 +19,9 @@ export const extractExifDataFromBlobPath = async ( }> => { const url = decodeURIComponent(blobPath); - const blobId = getIdFromBlobUrl(url); + const blobId = getIdFromStorageUrl(url); - const extension = getExtensionFromBlobUrl(url); + const extension = getExtensionFromStorageUrl(url); const fileBytes = blobPath ? await fetch(url) diff --git a/src/services/blob/index.ts b/src/services/blob/index.ts deleted file mode 100644 index fc9f6c17..00000000 --- a/src/services/blob/index.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { - VERCEL_BLOB_BASE_URL, - vercelBlobCopy, - vercelBlobDelete, - vercelBlobList, - vercelBlobUploadFromClient, -} from './vercel-blob'; -import { - AWS_S3_BASE_URL, - awsS3Copy, - awsS3Delete, - awsS3List, - awsS3UploadFromClient, - isUrlFromAwsS3, -} from './aws-s3'; -import { HAS_AWS_S3_STORAGE, HAS_AWS_S3_STORAGE_CLIENT } from '@/site/config'; - -const PREFIX_UPLOAD = 'upload'; -const PREFIX_PHOTO = 'photo'; -const BLOB_BASE_URL = HAS_AWS_S3_STORAGE_CLIENT - ? AWS_S3_BASE_URL - : VERCEL_BLOB_BASE_URL; - -const REGEX_UPLOAD_PATH = new RegExp( - `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`, - 'i', -); - -const REGEX_UPLOAD_ID = new RegExp( - `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`, - 'i', -); - -export const fileNameForBlobUrl = (url: string) => - url.replace(`${BLOB_BASE_URL}/`, ''); - -export const getExtensionFromBlobUrl = (url: string) => - url.match(/.([a-z]{1,4})$/i)?.[1]; - -export const getIdFromBlobUrl = (url: string) => - url.match(REGEX_UPLOAD_ID)?.[1]; - -export const isUploadPathnameValid = (pathname?: string) => - pathname?.match(REGEX_UPLOAD_PATH); - -const getFileNameFromBlobUrl = (url: string) => - (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? ''; - -export const uploadPhotoFromClient = async ( - file: File | Blob, - extension = 'jpg', -) => HAS_AWS_S3_STORAGE_CLIENT - ? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true) - : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`); - -export const convertUploadToPhoto = async ( - uploadUrl: string, - photoId?: string, -): Promise => { - const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`; - const fileExtension = getExtensionFromBlobUrl(uploadUrl); - const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`; - - const useAwsS3 = HAS_AWS_S3_STORAGE && isUrlFromAwsS3(uploadUrl); - - const url = await (useAwsS3 - ? awsS3Copy(uploadUrl, photoUrl, photoId === undefined) - : vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined)); - - if (url) { - await (useAwsS3 - ? awsS3Delete(getFileNameFromBlobUrl(uploadUrl)) - : vercelBlobDelete(uploadUrl)); - } - - return url; -}; - -export const deleteBlobUrl = (url: string) => - HAS_AWS_S3_STORAGE && isUrlFromAwsS3(url) - ? awsS3Delete(getFileNameFromBlobUrl(url)) - : vercelBlobDelete(url); - -export const getBlobUploadUrls = (): Promise => HAS_AWS_S3_STORAGE - ? awsS3List(`${PREFIX_UPLOAD}-`) - : vercelBlobList(`${PREFIX_UPLOAD}-`); - -export const getBlobPhotoUrls = (): Promise => HAS_AWS_S3_STORAGE - ? awsS3List(`${PREFIX_PHOTO}-`) - : vercelBlobList(`${PREFIX_PHOTO}-`); diff --git a/src/services/blob/aws-s3.ts b/src/services/storage/aws-s3.ts similarity index 73% rename from src/services/blob/aws-s3.ts rename to src/services/storage/aws-s3.ts index bff189ca..2ec518d5 100644 --- a/src/services/blob/aws-s3.ts +++ b/src/services/storage/aws-s3.ts @@ -1,4 +1,3 @@ -import { generateNanoid } from '@/utility/nanoid'; import { S3Client, CopyObjectCommand, @@ -6,13 +5,14 @@ import { ListObjectsCommand, PutObjectCommand, } from '@aws-sdk/client-s3'; +import { generateStorageId } from '.'; const AWS_S3_BUCKET = process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? ''; const AWS_S3_REGION = process.env.NEXT_PUBLIC_AWS_S3_REGION ?? ''; const AWS_S3_ACCESS_KEY = process.env.AWS_S3_ACCESS_KEY ?? ''; const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY ?? ''; - -const API_PATH_PRESIGNED_URL = '/api/aws-s3/presigned-url'; +export const AWS_S3_BASE_URL = + `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`; export const awsS3Client = () => new S3Client({ region: AWS_S3_REGION, @@ -22,36 +22,14 @@ export const awsS3Client = () => new S3Client({ }, }); -export const AWS_S3_BASE_URL = - `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`; +const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`; export const isUrlFromAwsS3 = (url: string) => url.startsWith(AWS_S3_BASE_URL); -const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`; - -const generateBlobId = () => generateNanoid(16); - export const awsS3PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' }); -export const awsS3UploadFromClient = async ( - file: File | Blob, - fileName: string, - extension: string, - addRandomSuffix?: boolean, -) => { - const key = addRandomSuffix - ? `${fileName}-${generateBlobId()}.${extension}` - : `${fileName}.${extension}`; - - const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`) - .then((response) => response.text()); - - return fetch(url, { method: 'PUT', body: file }) - .then(() => urlForKey(key)); -}; - export const awsS3Copy = async ( fileNameSource: string, fileNameDestination: string, @@ -60,7 +38,7 @@ export const awsS3Copy = async ( const name = fileNameSource.split('.')[0]; const extension = fileNameSource.split('.')[1]; const Key = addRandomSuffix - ? `${name}-${generateBlobId()}.${extension}` + ? `${name}-${generateStorageId()}.${extension}` : fileNameDestination; return awsS3Client().send(new CopyObjectCommand({ Bucket: AWS_S3_BUCKET, @@ -71,16 +49,16 @@ export const awsS3Copy = async ( .then(() => urlForKey(fileNameDestination)); }; -export const awsS3Delete = async (Key: string) => { - awsS3Client().send(new DeleteObjectCommand({ - Bucket: AWS_S3_BUCKET, - Key, - })); -}; - export const awsS3List = async (Prefix: string) => awsS3Client().send(new ListObjectsCommand({ Bucket: AWS_S3_BUCKET, Prefix, })) .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []); + +export const awsS3Delete = async (Key: string) => { + awsS3Client().send(new DeleteObjectCommand({ + Bucket: AWS_S3_BUCKET, + Key, + })); +}; diff --git a/src/services/storage/cloudflare-r2.ts b/src/services/storage/cloudflare-r2.ts new file mode 100644 index 00000000..7866005a --- /dev/null +++ b/src/services/storage/cloudflare-r2.ts @@ -0,0 +1,79 @@ +import { + S3Client, + ListObjectsCommand, + PutObjectCommand, + DeleteObjectCommand, + CopyObjectCommand, +} from '@aws-sdk/client-s3'; +import { generateStorageId } from '.'; + +const CLOUDFLARE_R2_BUCKET = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? ''; +const CLOUDFLARE_R2_ACCOUNT_ID = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? ''; +const CLOUDFLARE_R2_DEV_SUBDOMAIN = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? ''; +const CLOUDFLARE_R2_ACCESS_KEY = + process.env.CLOUDFLARE_R2_ACCESS_KEY ?? ''; +const CLOUDFLARE_R2_SECRET_ACCESS_KEY = + process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? ''; +const CLOUDFLARE_R2_ENDPOINT = + `https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`; + +export const CLOUDFLARE_R2_BASE_URL_PRIVATE = + `${CLOUDFLARE_R2_ENDPOINT}/${CLOUDFLARE_R2_BUCKET}`; + +export const CLOUDFLARE_R2_BASE_URL_PUBLIC = + `https://${CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev`; + +export const cloudflareR2Client = () => new S3Client({ + region: 'auto', + endpoint: CLOUDFLARE_R2_ENDPOINT, + credentials: { + accessKeyId: CLOUDFLARE_R2_ACCESS_KEY, + secretAccessKey: CLOUDFLARE_R2_SECRET_ACCESS_KEY, + }, +}); + +const urlForKey = (key?: string, isPublic = true) => isPublic + ? `${CLOUDFLARE_R2_BASE_URL_PUBLIC}/${key}` + : `${CLOUDFLARE_R2_BASE_URL_PRIVATE}/${key}`; + +export const isUrlFromCloudflareR2 = (url: string) => + url.startsWith(CLOUDFLARE_R2_BASE_URL_PRIVATE) || + url.startsWith(CLOUDFLARE_R2_BASE_URL_PUBLIC); + +export const cloudflareR2PutObjectCommandForKey = (Key: string) => + new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key }); + +export const cloudflareR2Copy = async ( + fileNameSource: string, + fileNameDestination: string, + addRandomSuffix?: boolean, +) => { + const name = fileNameSource.split('.')[0]; + const extension = fileNameSource.split('.')[1]; + const Key = addRandomSuffix + ? `${name}-${generateStorageId()}.${extension}` + : fileNameDestination; + return cloudflareR2Client().send(new CopyObjectCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + CopySource: `${CLOUDFLARE_R2_BUCKET}/${fileNameSource}`, + Key, + })) + .then(() => urlForKey(fileNameDestination)); +}; + +export const cloudflareR2List = async (Prefix: string) => + cloudflareR2Client().send(new ListObjectsCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + Prefix, + })) + .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []); + +export const cloudflareR2Delete = async (Key: string) => { + cloudflareR2Client().send(new DeleteObjectCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + Key, + })); +}; diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts new file mode 100644 index 00000000..2b58ebe3 --- /dev/null +++ b/src/services/storage/index.ts @@ -0,0 +1,205 @@ +import { + VERCEL_BLOB_BASE_URL, + vercelBlobCopy, + vercelBlobDelete, + vercelBlobList, + vercelBlobUploadFromClient, +} from './vercel-blob'; +import { + AWS_S3_BASE_URL, + awsS3Copy, + awsS3Delete, + awsS3List, + isUrlFromAwsS3, +} from './aws-s3'; +import { + CURRENT_STORAGE, + HAS_AWS_S3_STORAGE, + HAS_VERCEL_BLOB_STORAGE, + HAS_CLOUDFLARE_R2_STORAGE, +} from '@/site/config'; +import { generateNanoid } from '@/utility/nanoid'; +import { + CLOUDFLARE_R2_BASE_URL_PUBLIC, + cloudflareR2Copy, + cloudflareR2Delete, + cloudflareR2List, + isUrlFromCloudflareR2, +} from './cloudflare-r2'; +import { PATH_API_PRESIGNED_URL } from '@/site/paths'; + +export const generateStorageId = () => generateNanoid(16); + +export type StorageType = + 'vercel-blob' | + 'aws-s3' | + 'cloudflare-r2'; + +export const labelForStorage = (type: StorageType): string => { + switch (type) { + case 'vercel-blob': return 'Vercel Blob'; + case 'cloudflare-r2': return 'Cloudflare R2'; + case 'aws-s3': return 'AWS S3'; + } +}; + +export const baseUrlForStorage = (type: StorageType) => { + switch (type) { + case 'vercel-blob': return VERCEL_BLOB_BASE_URL; + case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC; + case 'aws-s3': return AWS_S3_BASE_URL; + } +}; + +export const storageTypeFromUrl = (url: string): StorageType => { + if (isUrlFromCloudflareR2(url)) { + return 'cloudflare-r2'; + } else if (isUrlFromAwsS3(url)) { + return 'aws-s3'; + } else { + return 'vercel-blob'; + } +}; + +const PREFIX_UPLOAD = 'upload'; +const PREFIX_PHOTO = 'photo'; + +const REGEX_UPLOAD_PATH = new RegExp( + `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`, + 'i', +); + +const REGEX_UPLOAD_ID = new RegExp( + `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`, + 'i', +); + +export const fileNameForStorageUrl = (url: string) => { + switch (storageTypeFromUrl(url)) { + case 'vercel-blob': + return url.replace(`${VERCEL_BLOB_BASE_URL}/`, ''); + case 'cloudflare-r2': + return url.replace(`${CLOUDFLARE_R2_BASE_URL_PUBLIC}/`, ''); + case 'aws-s3': + return url.replace(`${AWS_S3_BASE_URL}/`, ''); + } +}; + +export const getExtensionFromStorageUrl = (url: string) => + url.match(/.([a-z]{1,4})$/i)?.[1]; + +export const getIdFromStorageUrl = (url: string) => + url.match(REGEX_UPLOAD_ID)?.[1]; + +export const isUploadPathnameValid = (pathname?: string) => + pathname?.match(REGEX_UPLOAD_PATH); + +const getFileNameFromStorageUrl = (url: string) => + (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? ''; + +export const uploadFromClientViaPresignedUrl = async ( + file: File | Blob, + fileName: string, + extension: string, + addRandomSuffix?: boolean, +) => { + const key = addRandomSuffix + ? `${fileName}-${generateStorageId()}.${extension}` + : `${fileName}.${extension}`; + + const url = await fetch(`${PATH_API_PRESIGNED_URL}/${key}`) + .then((response) => response.text()); + + return fetch(url, { method: 'PUT', body: file }) + .then(() => `${baseUrlForStorage(CURRENT_STORAGE)}/${key}`); +}; + +export const uploadPhotoFromClient = async ( + file: File | Blob, + extension = 'jpg', +) => ( + CURRENT_STORAGE === 'cloudflare-r2' || + CURRENT_STORAGE === 'aws-s3' +) + ? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true) + : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`); + +export const convertUploadToPhoto = async ( + uploadUrl: string, + photoId?: string, +): Promise => { + const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`; + const fileExtension = getExtensionFromStorageUrl(uploadUrl); + const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`; + + const storageType = storageTypeFromUrl(uploadUrl); + + let url: string | undefined; + + // Copy file + switch (storageType) { + case 'vercel-blob': + url = await vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined); + break; + case 'cloudflare-r2': + url = await cloudflareR2Copy( + getFileNameFromStorageUrl(uploadUrl), + photoUrl, + photoId === undefined, + ); + break; + case 'aws-s3': + url = await awsS3Copy(uploadUrl, photoUrl, photoId === undefined); + break; + } + + // If successful, delete original file + if (url) { + switch (storageType) { + case 'vercel-blob': + await vercelBlobDelete(uploadUrl); + break; + case 'cloudflare-r2': + await cloudflareR2Delete(getFileNameFromStorageUrl(uploadUrl)); + break; + case 'aws-s3': + await awsS3Delete(getFileNameFromStorageUrl(uploadUrl)); + break; + } + } + + return url; +}; + +export const deleteStorageUrl = (url: string) => { + switch (storageTypeFromUrl(url)) { + case 'vercel-blob': + return vercelBlobDelete(url); + case 'cloudflare-r2': + return cloudflareR2Delete(getFileNameFromStorageUrl(url)); + case 'aws-s3': + return awsS3Delete(getFileNameFromStorageUrl(url)); + } +}; + +const getStorageUrlsForPrefix = async (prefix = ''): Promise => { + const urls: string[] = []; + + if (HAS_VERCEL_BLOB_STORAGE) { + urls.push(...await vercelBlobList(prefix)); + } + if (HAS_AWS_S3_STORAGE) { + urls.push(...await awsS3List(prefix)); + } + if (HAS_CLOUDFLARE_R2_STORAGE) { + urls.push(...await cloudflareR2List(prefix)); + } + + return urls; +}; + +export const getStorageUploadUrls = () => + getStorageUrlsForPrefix(`${PREFIX_UPLOAD}-`); + +export const getStoragePhotoUrls = () => + getStorageUrlsForPrefix(`${PREFIX_PHOTO}-`); diff --git a/src/services/blob/vercel-blob.ts b/src/services/storage/vercel-blob.ts similarity index 83% rename from src/services/blob/vercel-blob.ts rename to src/services/storage/vercel-blob.ts index f14b2041..a8b5094f 100644 --- a/src/services/blob/vercel-blob.ts +++ b/src/services/storage/vercel-blob.ts @@ -1,4 +1,4 @@ -import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths'; +import { PATH_API_VERCEL_BLOB_UPLOAD } from '@/site/paths'; import { copy, del, list } from '@vercel/blob'; import { upload } from '@vercel/blob/client'; @@ -9,6 +9,9 @@ const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match( export const VERCEL_BLOB_BASE_URL = `https://${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`; +export const isUrlFromVercelBlob = (url: string) => + url.startsWith(VERCEL_BLOB_BASE_URL); + export const vercelBlobUploadFromClient = async ( file: File | Blob, fileName: string, @@ -18,7 +21,7 @@ export const vercelBlobUploadFromClient = async ( file, { access: 'public', - handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB, + handleUploadUrl: PATH_API_VERCEL_BLOB_UPLOAD, }, ) .then(({ url }) => url); diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 80484739..6ea9004e 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -19,12 +19,16 @@ import Checklist from '@/components/Checklist'; import { toastSuccess } from '@/toast'; import { ConfigChecklistStatus } from './config'; import StatusIcon from '@/components/StatusIcon'; +import { labelForStorage } from '@/services/storage'; export default function SiteChecklistClient({ hasPostgres, - hasBlob, - hasVercelBlob, + hasStorage, + hasVercelBlobStorage, + hasCloudflareR2Storage, hasAwsS3Storage, + hasMultipleStorageProviders, + currentStorage, hasAuth, hasAdminUser, hasTitle, @@ -36,8 +40,8 @@ export default function SiteChecklistClient({ isPriorityOrderEnabled, isPublicApiEnabled, isOgTextBottomAligned, - showRefreshButton, gridAspectRatio, + showRefreshButton, secret, }: ConfigChecklistStatus & { showRefreshButton?: boolean @@ -139,14 +143,18 @@ export default function SiteChecklistClient({ and connect to project {renderSubStatus( - hasVercelBlob ? 'checked' : 'optional', + hasVercelBlobStorage ? 'checked' : 'optional', <> - Vercel Blob: + {labelForStorage('vercel-blob')}: {' '} {renderLink( // eslint-disable-next-line max-len @@ -157,10 +165,21 @@ export default function SiteChecklistClient({ and connect to project , )} + {renderSubStatus( + hasCloudflareR2Storage ? 'checked' : 'optional', + <> + {labelForStorage('cloudflare-r2')}: + {' '} + {renderLink( + 'https://github.com/sambecker/exif-photo-blog#cloudflare-r2', + 'create/configure bucket', + )} + + )} {renderSubStatus( hasAwsS3Storage ? 'checked' : 'optional', <> - AWS S3: + {labelForStorage('aws-s3')}: {' '} {renderLink( 'https://github.com/sambecker/exif-photo-blog#aws-s3', diff --git a/src/site/config.ts b/src/site/config.ts index 4cfb07be..c8f131f1 100644 --- a/src/site/config.ts +++ b/src/site/config.ts @@ -1,3 +1,4 @@ +import type { StorageType } from '@/services/storage'; import { makeUrlAbsolute, shortenUrl } from '@/utility/url'; // META / DOMAINS @@ -31,12 +32,22 @@ export const BASE_URL = process.env.NODE_ENV === 'production' : 'http://localhost:3000'; // STORAGE: VERCEL BLOB -export const HAS_VERCEL_BLOB = +export const HAS_VERCEL_BLOB_STORAGE = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0; +// STORAGE: Cloudflare R2 +// Includes separate check for client-side usage, i.e., url construction +export const HAS_CLOUDFLARE_R2_STORAGE_CLIENT = + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? '').length > 0 && + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? '').length > 0 && + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? '').length > 0; +export const HAS_CLOUDFLARE_R2_STORAGE = + HAS_CLOUDFLARE_R2_STORAGE_CLIENT && + (process.env.CLOUDFLARE_R2_ACCESS_KEY ?? '').length > 0 && + (process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? '').length > 0; + // STORAGE: AWS S3 -// Includes separate check for client-side usage, -// i.e., uploading, url construction +// Includes separate check for client-side usage, i.e., url construction export const HAS_AWS_S3_STORAGE_CLIENT = (process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? '').length > 0 && (process.env.NEXT_PUBLIC_AWS_S3_REGION ?? '').length > 0; @@ -45,6 +56,23 @@ export const HAS_AWS_S3_STORAGE = (process.env.AWS_S3_ACCESS_KEY ?? '').length > 0 && (process.env.AWS_S3_SECRET_ACCESS_KEY ?? '').length > 0; +export const HAS_MULTIPLE_STORAGE_PROVIDERS = [ + HAS_VERCEL_BLOB_STORAGE, + HAS_CLOUDFLARE_R2_STORAGE, + HAS_AWS_S3_STORAGE, +].filter(Boolean).length > 1; + +// Storage preference requires client-available keys +// so it can be reached in the browser when uploading +export const CURRENT_STORAGE: StorageType = + (process.env.NEXT_PUBLIC_STORAGE_PREFERENCE as StorageType | undefined) || ( + HAS_CLOUDFLARE_R2_STORAGE_CLIENT + ? 'cloudflare-r2' + : HAS_AWS_S3_STORAGE_CLIENT + ? 'aws-s3' + : 'vercel-blob' + ); + // SETTINGS export const PRO_MODE_ENABLED = process.env.NEXT_PUBLIC_PRO_MODE === '1'; @@ -65,9 +93,15 @@ export const HIGH_DENSITY_GRID = GRID_ASPECT_RATIO <= 1; export const CONFIG_CHECKLIST_STATUS = { hasPostgres: (process.env.POSTGRES_HOST ?? '').length > 0, - hasBlob: HAS_VERCEL_BLOB || HAS_AWS_S3_STORAGE, - hasVercelBlob: HAS_VERCEL_BLOB, + hasVercelBlobStorage: HAS_VERCEL_BLOB_STORAGE, + hasCloudflareR2Storage: HAS_CLOUDFLARE_R2_STORAGE, hasAwsS3Storage: HAS_AWS_S3_STORAGE, + hasStorage: + HAS_VERCEL_BLOB_STORAGE || + HAS_CLOUDFLARE_R2_STORAGE || + HAS_AWS_S3_STORAGE, + hasMultipleStorageProviders: HAS_MULTIPLE_STORAGE_PROVIDERS, + currentStorage: CURRENT_STORAGE, hasAuth: (process.env.AUTH_SECRET ?? '').length > 0, hasAdminUser: ( (process.env.ADMIN_EMAIL ?? '').length > 0 && @@ -89,6 +123,6 @@ export type ConfigChecklistStatus = typeof CONFIG_CHECKLIST_STATUS; export const IS_SITE_READY = CONFIG_CHECKLIST_STATUS.hasPostgres && - CONFIG_CHECKLIST_STATUS.hasBlob && + CONFIG_CHECKLIST_STATUS.hasStorage && CONFIG_CHECKLIST_STATUS.hasAuth && CONFIG_CHECKLIST_STATUS.hasAdminUser; diff --git a/src/site/paths.ts b/src/site/paths.ts index 77c8d9f2..e6b699ce 100644 --- a/src/site/paths.ts +++ b/src/site/paths.ts @@ -12,6 +12,7 @@ export const PATH_ROOT = '/'; export const PATH_GRID = '/grid'; export const PATH_SETS = '/sets'; export const PATH_ADMIN = '/admin'; +export const PATH_API = '/api'; export const PATH_SIGN_IN = '/sign-in'; export const PATH_OG = '/og'; @@ -31,9 +32,13 @@ const PATH_FILM_SIMULATION_DYNAMIC = `${PREFIX_FILM_SIMULATION}/[simulation]`; export const PATH_ADMIN_PHOTOS = `${PATH_ADMIN}/photos`; export const PATH_ADMIN_UPLOADS = `${PATH_ADMIN}/uploads`; export const PATH_ADMIN_TAGS = `${PATH_ADMIN}/tags`; -export const PATH_ADMIN_UPLOAD_BLOB = `${PATH_ADMIN_UPLOADS}/blob`; export const PATH_ADMIN_CONFIGURATION = `${PATH_ADMIN}/configuration`; +// API paths +export const PATH_API_STORAGE = `${PATH_API}/storage`; +export const PATH_API_VERCEL_BLOB_UPLOAD = `${PATH_API_STORAGE}/vercel-blob`; +export const PATH_API_PRESIGNED_URL = `${PATH_API_STORAGE}/presigned-url`; + // Modifiers const SHARE = 'share'; const NEXT = 'next'; @@ -44,7 +49,6 @@ export const PATHS_ADMIN = [ PATH_ADMIN_PHOTOS, PATH_ADMIN_UPLOADS, PATH_ADMIN_TAGS, - PATH_ADMIN_UPLOAD_BLOB, PATH_ADMIN_CONFIGURATION, ];