From 16c524abc48aaa074ce38d134d6c6b33003618d3 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sat, 20 Jan 2024 22:13:05 -0600 Subject: [PATCH 1/9] Add support for Cloudflare R2 storage --- .vscode/settings.json | 1 + README.md | 37 ++++++++++- next.config.js | 6 ++ src/admin/BlobUrls.tsx | 2 +- src/app/admin/uploads/blob/route.tsx | 2 +- .../presigned-url/[key]/route.ts | 15 ++++- src/cache/index.ts | 2 +- src/photo/PhotoUpload.tsx | 2 +- src/photo/actions.ts | 2 +- src/photo/server.ts | 2 +- src/services/{blob => storage}/aws-s3.ts | 28 +-------- src/services/storage/cloudflare-r2.ts | 32 ++++++++++ src/services/{blob => storage}/index.ts | 62 ++++++++++++++++--- src/services/{blob => storage}/vercel-blob.ts | 0 src/site/SiteChecklistClient.tsx | 31 +++++++--- src/site/config.ts | 39 ++++++++++-- 16 files changed, 208 insertions(+), 55 deletions(-) rename src/app/api/{aws-s3 => storage}/presigned-url/[key]/route.ts (57%) rename src/services/{blob => storage}/aws-s3.ts (75%) create mode 100644 src/services/storage/cloudflare-r2.ts rename src/services/{blob => storage}/index.ts (59%) rename src/services/{blob => storage}/vercel-blob.ts (100%) diff --git a/.vscode/settings.json b/.vscode/settings.json index e664ce2a..84c45555 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,7 @@ "ARROWRIGHT", "Astia", "camelcase", + "cloudflarestorage", "CredentialsSignin", "Eterna", "exif", diff --git a/README.md b/README.md index 919f2553..bcdaf2d8 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,42 @@ Installation - `NEXT_PUBLIC_GRID_ASPECT_RATIO = 1.5` sets aspect ratio for grid tiles (defaults to `1`—setting to `0` removes the constraint) - `NEXT_PUBLIC_OG_TEXT_ALIGNMENT = BOTTOM` keeps OG image text bottom aligned (default is top) -### Setup alternate storage +### Configure alternate storage + +Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations). + +If you have multiple adapters, you can set one as preferred by storing `aws-s3`, `cloudflare-r2`, or `vercel-blob` in `NEXT_PUBLIC_STORAGE_PREFERENCE`. + +#### Cloudflare R2 + +1. Setup bucket + - [Create R2 bucket](https://developers.cloudflare.com/r2/) with default settings + - Setup CORS under bucket settings: + ```json + [{ + "AllowedHeaders": ["*"] + "AllowedOrigins": [ + "http://localhost:3000", + "https://{VERCEL_PROJECT_NAME}*.vercel.app", + "{PRODUCTION_DOMAIN}" + ], + "AllowedMethods": [ + "GET", + "PUT" + ], + }] + ``` + - Enable R2.dev subdomain (necessary in order to serve files publicly without a custom domain) + - Store configuration: + - `NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET`: bucket name + - `NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID`: account id (found on R2 overview page) + - `NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN`: r2.dev subdomain, e.g., "pub-jf90908..." +2. Setup credentials + - Create API token by selecting "Manage R2 API Tokens," and clicking "Create API Token" + - Select "Object Read & Write," choose "Apply to specific buckets only," and select the bucket created in Step 1. + - Store credentials (⚠️ _Ensure access keys are not prefixed with `NEXT_PUBLIC`_): + - `CLOUDFLARE_R2_ACCESS_KEY` + - `CLOUDFLARE_R2_SECRET_ACCESS_KEY` #### AWS S3 diff --git a/next.config.js b/next.config.js index 9ee18918..bd880f53 100644 --- a/next.config.js +++ b/next.config.js @@ -6,6 +6,11 @@ const VERCEL_BLOB_HOSTNAME = VERCEL_BLOB_STORE_ID ? `${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com` : undefined; +const CLOUDFLARE_R2_HOSTNAME = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN + ? `${process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev` + : undefined; + const AWS_S3_HOSTNAME = process.env.NEXT_PUBLIC_AWS_S3_BUCKET && process.env.NEXT_PUBLIC_AWS_S3_REGION @@ -28,6 +33,7 @@ const nextConfig = { imageSizes: [200], remotePatterns: [] .concat(createRemotePattern(VERCEL_BLOB_HOSTNAME)) + .concat(createRemotePattern(CLOUDFLARE_R2_HOSTNAME)) .concat(createRemotePattern(AWS_S3_HOSTNAME)), minimumCacheTTL: 31536000, }, diff --git a/src/admin/BlobUrls.tsx b/src/admin/BlobUrls.tsx index 29020f8e..bed92015 100644 --- a/src/admin/BlobUrls.tsx +++ b/src/admin/BlobUrls.tsx @@ -2,7 +2,7 @@ import { Fragment } from 'react'; import AdminGrid from './AdminGrid'; import Link from 'next/link'; import ImageTiny from '@/components/ImageTiny'; -import { fileNameForBlobUrl } from '@/services/blob'; +import { fileNameForBlobUrl } from '@/services/storage'; import FormWithConfirm from '@/components/FormWithConfirm'; import { deleteBlobPhotoAction } from '@/photo/actions'; import DeleteButton from './DeleteButton'; diff --git a/src/app/admin/uploads/blob/route.tsx b/src/app/admin/uploads/blob/route.tsx index 22b14c3d..a6436018 100644 --- a/src/app/admin/uploads/blob/route.tsx +++ b/src/app/admin/uploads/blob/route.tsx @@ -4,7 +4,7 @@ import { ACCEPTED_PHOTO_FILE_TYPES, MAX_PHOTO_UPLOAD_SIZE_IN_BYTES, } from '@/photo'; -import { isUploadPathnameValid } from '@/services/blob'; +import { isUploadPathnameValid } from '@/services/storage'; import { handleUpload, type HandleUploadBody } from '@vercel/blob/client'; import { NextResponse } from 'next/server'; diff --git a/src/app/api/aws-s3/presigned-url/[key]/route.ts b/src/app/api/storage/presigned-url/[key]/route.ts similarity index 57% rename from src/app/api/aws-s3/presigned-url/[key]/route.ts rename to src/app/api/storage/presigned-url/[key]/route.ts index e2e858fe..2b77455d 100644 --- a/src/app/api/aws-s3/presigned-url/[key]/route.ts +++ b/src/app/api/storage/presigned-url/[key]/route.ts @@ -2,7 +2,12 @@ import { auth } from '@/auth'; import { awsS3Client, awsS3PutObjectCommandForKey, -} from '@/services/blob/aws-s3'; +} from '@/services/storage/aws-s3'; +import { + cloudflareR2Client, + cloudflareR2PutObjectCommandForKey, +} from '@/services/storage/cloudflare-r2'; +import { STORAGE_PREFERENCE } from '@/site/config'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const runtime = 'edge'; @@ -14,8 +19,12 @@ export async function GET( const session = await auth(); if (session?.user && key) { const url = await getSignedUrl( - awsS3Client(), - awsS3PutObjectCommandForKey(key), + STORAGE_PREFERENCE === 'cloudflare-r2' + ? cloudflareR2Client() + : awsS3Client(), + STORAGE_PREFERENCE === 'cloudflare-r2' + ? cloudflareR2PutObjectCommandForKey(key) + : awsS3PutObjectCommandForKey(key), { expiresIn: 3600 } ); return new Response( diff --git a/src/cache/index.ts b/src/cache/index.ts index ce35f63b..d67f3ade 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -24,7 +24,7 @@ import { getPhotosNearId, } from '@/services/vercel-postgres'; import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo'; -import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob'; +import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/storage'; import type { Session } from 'next-auth'; import { createCameraKey } from '@/camera'; import { PATHS_ADMIN } from '@/site/paths'; diff --git a/src/photo/PhotoUpload.tsx b/src/photo/PhotoUpload.tsx index 71d6a12f..b50fcdff 100644 --- a/src/photo/PhotoUpload.tsx +++ b/src/photo/PhotoUpload.tsx @@ -1,7 +1,7 @@ 'use client'; import { useState } from 'react'; -import { uploadPhotoFromClient } from '@/services/blob'; +import { uploadPhotoFromClient } from '@/services/storage'; import { useRouter } from 'next/navigation'; import { PATH_ADMIN_UPLOADS, pathForAdminUploadUrl } from '@/site/paths'; import ImageInput from '../components/ImageInput'; diff --git a/src/photo/actions.ts b/src/photo/actions.ts index f017d0f4..1aa5ef2a 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -17,7 +17,7 @@ import { redirect } from 'next/navigation'; import { convertUploadToPhoto, deleteBlobUrl, -} from '@/services/blob'; +} from '@/services/storage'; import { revalidateAdminPaths, revalidateAllKeysAndPaths, diff --git a/src/photo/server.ts b/src/photo/server.ts index 7971c917..d0887746 100644 --- a/src/photo/server.ts +++ b/src/photo/server.ts @@ -1,7 +1,7 @@ import { getExtensionFromBlobUrl, getIdFromBlobUrl, -} from '@/services/blob'; +} from '@/services/storage'; import { convertExifToFormData } from '@/photo/form'; import { getFujifilmSimulationFromMakerNote, diff --git a/src/services/blob/aws-s3.ts b/src/services/storage/aws-s3.ts similarity index 75% rename from src/services/blob/aws-s3.ts rename to src/services/storage/aws-s3.ts index bff189ca..4ce72ee5 100644 --- a/src/services/blob/aws-s3.ts +++ b/src/services/storage/aws-s3.ts @@ -1,4 +1,3 @@ -import { generateNanoid } from '@/utility/nanoid'; import { S3Client, CopyObjectCommand, @@ -6,13 +5,14 @@ import { ListObjectsCommand, PutObjectCommand, } from '@aws-sdk/client-s3'; +import { generateBlobId } from '.'; const AWS_S3_BUCKET = process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? ''; const AWS_S3_REGION = process.env.NEXT_PUBLIC_AWS_S3_REGION ?? ''; const AWS_S3_ACCESS_KEY = process.env.AWS_S3_ACCESS_KEY ?? ''; const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY ?? ''; - -const API_PATH_PRESIGNED_URL = '/api/aws-s3/presigned-url'; +export const AWS_S3_BASE_URL = + `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`; export const awsS3Client = () => new S3Client({ region: AWS_S3_REGION, @@ -22,36 +22,14 @@ export const awsS3Client = () => new S3Client({ }, }); -export const AWS_S3_BASE_URL = - `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`; - export const isUrlFromAwsS3 = (url: string) => url.startsWith(AWS_S3_BASE_URL); const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`; -const generateBlobId = () => generateNanoid(16); - export const awsS3PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' }); -export const awsS3UploadFromClient = async ( - file: File | Blob, - fileName: string, - extension: string, - addRandomSuffix?: boolean, -) => { - const key = addRandomSuffix - ? `${fileName}-${generateBlobId()}.${extension}` - : `${fileName}.${extension}`; - - const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`) - .then((response) => response.text()); - - return fetch(url, { method: 'PUT', body: file }) - .then(() => urlForKey(key)); -}; - export const awsS3Copy = async ( fileNameSource: string, fileNameDestination: string, diff --git a/src/services/storage/cloudflare-r2.ts b/src/services/storage/cloudflare-r2.ts new file mode 100644 index 00000000..1595e4d1 --- /dev/null +++ b/src/services/storage/cloudflare-r2.ts @@ -0,0 +1,32 @@ +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; + +const CLOUDFLARE_R2_BUCKET = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? ''; +const CLOUDFLARE_R2_ACCOUNT_ID = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? ''; +const CLOUDFLARE_R2_DEV_SUBDOMAIN = + process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? ''; +const CLOUDFLARE_R2_ACCESS_KEY = + process.env.CLOUDFLARE_R2_ACCESS_KEY ?? ''; +const CLOUDFLARE_R2_SECRET_ACCESS_KEY = + process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? ''; +const CLOUDFLARE_R2_ENDPOINT = + `https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`; + +export const CLOUDFLARE_R2_BASE_URL_PRIVATE = + `${CLOUDFLARE_R2_ENDPOINT}/${CLOUDFLARE_R2_BUCKET}`; + +export const CLOUDFLARE_R2_BASE_URL_PUBLIC = + `https://${CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev`; + +export const cloudflareR2Client = () => new S3Client({ + region: 'auto', + endpoint: CLOUDFLARE_R2_ENDPOINT, + credentials: { + accessKeyId: CLOUDFLARE_R2_ACCESS_KEY, + secretAccessKey: CLOUDFLARE_R2_SECRET_ACCESS_KEY, + }, +}); + +export const cloudflareR2PutObjectCommandForKey = (Key: string) => + new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key }); diff --git a/src/services/blob/index.ts b/src/services/storage/index.ts similarity index 59% rename from src/services/blob/index.ts rename to src/services/storage/index.ts index fc9f6c17..5917cfbc 100644 --- a/src/services/blob/index.ts +++ b/src/services/storage/index.ts @@ -10,16 +10,44 @@ import { awsS3Copy, awsS3Delete, awsS3List, - awsS3UploadFromClient, isUrlFromAwsS3, } from './aws-s3'; -import { HAS_AWS_S3_STORAGE, HAS_AWS_S3_STORAGE_CLIENT } from '@/site/config'; +import { + STORAGE_PREFERENCE, + HAS_AWS_S3_STORAGE, +} from '@/site/config'; +import { generateNanoid } from '@/utility/nanoid'; +import { CLOUDFLARE_R2_BASE_URL_PUBLIC } from './cloudflare-r2'; + +export const generateBlobId = () => generateNanoid(16); + +export type StorageType = + 'vercel-blob' | + 'aws-s3' | + 'cloudflare-r2'; + +export const labelForStorage = (type: StorageType): string => { + switch (type) { + case 'vercel-blob': return 'Vercel Blob'; + case 'cloudflare-r2': return 'Cloudflare R2'; + case 'aws-s3': return 'AWS S3'; + } +}; + +const blobBaseUrlForStorage = (type: StorageType) => { + switch (type) { + case 'vercel-blob': return VERCEL_BLOB_BASE_URL; + case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC; + case 'aws-s3': return AWS_S3_BASE_URL; + } +}; + +export const BLOB_BASE_URL = blobBaseUrlForStorage(STORAGE_PREFERENCE); + +const API_PATH_PRESIGNED_URL = '/api/storage/presigned-url'; const PREFIX_UPLOAD = 'upload'; const PREFIX_PHOTO = 'photo'; -const BLOB_BASE_URL = HAS_AWS_S3_STORAGE_CLIENT - ? AWS_S3_BASE_URL - : VERCEL_BLOB_BASE_URL; const REGEX_UPLOAD_PATH = new RegExp( `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`, @@ -46,11 +74,31 @@ export const isUploadPathnameValid = (pathname?: string) => const getFileNameFromBlobUrl = (url: string) => (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? ''; +export const uploadFromClientViaPresignedUrl = async ( + file: File | Blob, + fileName: string, + extension: string, + addRandomSuffix?: boolean, +) => { + const key = addRandomSuffix + ? `${fileName}-${generateBlobId()}.${extension}` + : `${fileName}.${extension}`; + + const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`) + .then((response) => response.text()); + + return fetch(url, { method: 'PUT', body: file }) + .then(() => `${BLOB_BASE_URL}/${key}`); +}; + export const uploadPhotoFromClient = async ( file: File | Blob, extension = 'jpg', -) => HAS_AWS_S3_STORAGE_CLIENT - ? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true) +) => ( + STORAGE_PREFERENCE === 'cloudflare-r2' || + STORAGE_PREFERENCE === 'aws-s3' +) + ? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true) : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`); export const convertUploadToPhoto = async ( diff --git a/src/services/blob/vercel-blob.ts b/src/services/storage/vercel-blob.ts similarity index 100% rename from src/services/blob/vercel-blob.ts rename to src/services/storage/vercel-blob.ts diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 80484739..9417c5d8 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -19,12 +19,15 @@ import Checklist from '@/components/Checklist'; import { toastSuccess } from '@/toast'; import { ConfigChecklistStatus } from './config'; import StatusIcon from '@/components/StatusIcon'; +import { labelForStorage } from '@/services/storage'; export default function SiteChecklistClient({ hasPostgres, - hasBlob, - hasVercelBlob, + hasStorage, + hasVercelBlobStorage, + hasCloudflareR2Storage, hasAwsS3Storage, + storagePreference, hasAuth, hasAdminUser, hasTitle, @@ -139,14 +142,17 @@ export default function SiteChecklistClient({ and connect to project {renderSubStatus( - hasVercelBlob ? 'checked' : 'optional', + hasVercelBlobStorage ? 'checked' : 'optional', <> - Vercel Blob: + {labelForStorage('vercel-blob')}: {' '} {renderLink( // eslint-disable-next-line max-len @@ -157,10 +163,21 @@ export default function SiteChecklistClient({ and connect to project , )} + {renderSubStatus( + hasCloudflareR2Storage ? 'checked' : 'optional', + <> + {labelForStorage('cloudflare-r2')}: + {' '} + {renderLink( + 'https://github.com/sambecker/exif-photo-blog#cloudflare-r2', + 'create/configure bucket', + )} + + )} {renderSubStatus( hasAwsS3Storage ? 'checked' : 'optional', <> - AWS S3: + {labelForStorage('aws-s3')}: {' '} {renderLink( 'https://github.com/sambecker/exif-photo-blog#aws-s3', diff --git a/src/site/config.ts b/src/site/config.ts index 4cfb07be..1cc4b59e 100644 --- a/src/site/config.ts +++ b/src/site/config.ts @@ -1,3 +1,4 @@ +import type { StorageType } from '@/services/storage'; import { makeUrlAbsolute, shortenUrl } from '@/utility/url'; // META / DOMAINS @@ -31,12 +32,22 @@ export const BASE_URL = process.env.NODE_ENV === 'production' : 'http://localhost:3000'; // STORAGE: VERCEL BLOB -export const HAS_VERCEL_BLOB = +export const HAS_VERCEL_BLOB_STORAGE = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0; +// STORAGE: Cloudflare R2 +// Includes separate check for client-side usage, i.e., url construction +export const HAS_CLOUDFLARE_R2_STORAGE_CLIENT = + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? '').length > 0 && + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? '').length > 0 && + (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? '').length > 0; +export const HAS_CLOUDFLARE_R2_STORAGE = + HAS_CLOUDFLARE_R2_STORAGE_CLIENT && + (process.env.CLOUDFLARE_R2_ACCESS_KEY ?? '').length > 0 && + (process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? '').length > 0; + // STORAGE: AWS S3 -// Includes separate check for client-side usage, -// i.e., uploading, url construction +// Includes separate check for client-side usage, i.e., url construction export const HAS_AWS_S3_STORAGE_CLIENT = (process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? '').length > 0 && (process.env.NEXT_PUBLIC_AWS_S3_REGION ?? '').length > 0; @@ -45,6 +56,17 @@ export const HAS_AWS_S3_STORAGE = (process.env.AWS_S3_ACCESS_KEY ?? '').length > 0 && (process.env.AWS_S3_SECRET_ACCESS_KEY ?? '').length > 0; +// Storage preference relies on client-only keys +// so that it's available in the browser when uploading +export const STORAGE_PREFERENCE: StorageType = + (process.env.NEXT_PUBLIC_STORAGE_PREFERENCE as StorageType | undefined) || ( + HAS_CLOUDFLARE_R2_STORAGE_CLIENT + ? 'cloudflare-r2' + : HAS_AWS_S3_STORAGE_CLIENT + ? 'aws-s3' + : 'vercel-blob' + ); + // SETTINGS export const PRO_MODE_ENABLED = process.env.NEXT_PUBLIC_PRO_MODE === '1'; @@ -65,9 +87,14 @@ export const HIGH_DENSITY_GRID = GRID_ASPECT_RATIO <= 1; export const CONFIG_CHECKLIST_STATUS = { hasPostgres: (process.env.POSTGRES_HOST ?? '').length > 0, - hasBlob: HAS_VERCEL_BLOB || HAS_AWS_S3_STORAGE, - hasVercelBlob: HAS_VERCEL_BLOB, + hasVercelBlobStorage: HAS_VERCEL_BLOB_STORAGE, + hasCloudflareR2Storage: HAS_CLOUDFLARE_R2_STORAGE, hasAwsS3Storage: HAS_AWS_S3_STORAGE, + hasStorage: + HAS_VERCEL_BLOB_STORAGE || + HAS_CLOUDFLARE_R2_STORAGE || + HAS_AWS_S3_STORAGE, + storagePreference: STORAGE_PREFERENCE, hasAuth: (process.env.AUTH_SECRET ?? '').length > 0, hasAdminUser: ( (process.env.ADMIN_EMAIL ?? '').length > 0 && @@ -89,6 +116,6 @@ export type ConfigChecklistStatus = typeof CONFIG_CHECKLIST_STATUS; export const IS_SITE_READY = CONFIG_CHECKLIST_STATUS.hasPostgres && - CONFIG_CHECKLIST_STATUS.hasBlob && + CONFIG_CHECKLIST_STATUS.hasStorage && CONFIG_CHECKLIST_STATUS.hasAuth && CONFIG_CHECKLIST_STATUS.hasAdminUser; From 2d244faec6198f6de31a94b612edc98aa6d5a62f Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sat, 20 Jan 2024 22:14:20 -0600 Subject: [PATCH 2/9] Update README storage --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index bcdaf2d8..27f5dec3 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,7 @@ Installation ### Configure alternate storage -Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations). - -If you have multiple adapters, you can set one as preferred by storing `aws-s3`, `cloudflare-r2`, or `vercel-blob` in `NEXT_PUBLIC_STORAGE_PREFERENCE`. +Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations). If you have multiple adapters, you can set one as preferred by storing "aws-s3," "cloudflare-r2," or "vercel-blob" in `NEXT_PUBLIC_STORAGE_PREFERENCE`. #### Cloudflare R2 From 5176649ad682bd6fcb54027e37c6902c3eb44555 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sat, 20 Jan 2024 22:39:23 -0600 Subject: [PATCH 3/9] Refactor storage api paths --- .../blob/route.tsx => api/storage/vercel-blob/route.ts} | 2 +- src/services/storage/index.ts | 5 ++--- src/services/storage/vercel-blob.ts | 4 ++-- src/site/paths.ts | 8 ++++++-- 4 files changed, 11 insertions(+), 8 deletions(-) rename src/app/{admin/uploads/blob/route.tsx => api/storage/vercel-blob/route.ts} (96%) diff --git a/src/app/admin/uploads/blob/route.tsx b/src/app/api/storage/vercel-blob/route.ts similarity index 96% rename from src/app/admin/uploads/blob/route.tsx rename to src/app/api/storage/vercel-blob/route.ts index a6436018..e41543c6 100644 --- a/src/app/admin/uploads/blob/route.tsx +++ b/src/app/api/storage/vercel-blob/route.ts @@ -9,7 +9,7 @@ import { handleUpload, type HandleUploadBody } from '@vercel/blob/client'; import { NextResponse } from 'next/server'; export async function POST(request: Request): Promise { - const body = (await request.json()) as HandleUploadBody; + const body: HandleUploadBody = await request.json(); try { const jsonResponse = await handleUpload({ diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 5917cfbc..8e7b0810 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -18,6 +18,7 @@ import { } from '@/site/config'; import { generateNanoid } from '@/utility/nanoid'; import { CLOUDFLARE_R2_BASE_URL_PUBLIC } from './cloudflare-r2'; +import { PATH_API_PRESIGNED_URL } from '@/site/paths'; export const generateBlobId = () => generateNanoid(16); @@ -44,8 +45,6 @@ const blobBaseUrlForStorage = (type: StorageType) => { export const BLOB_BASE_URL = blobBaseUrlForStorage(STORAGE_PREFERENCE); -const API_PATH_PRESIGNED_URL = '/api/storage/presigned-url'; - const PREFIX_UPLOAD = 'upload'; const PREFIX_PHOTO = 'photo'; @@ -84,7 +83,7 @@ export const uploadFromClientViaPresignedUrl = async ( ? `${fileName}-${generateBlobId()}.${extension}` : `${fileName}.${extension}`; - const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`) + const url = await fetch(`${PATH_API_PRESIGNED_URL}/${key}`) .then((response) => response.text()); return fetch(url, { method: 'PUT', body: file }) diff --git a/src/services/storage/vercel-blob.ts b/src/services/storage/vercel-blob.ts index f14b2041..5a664f6b 100644 --- a/src/services/storage/vercel-blob.ts +++ b/src/services/storage/vercel-blob.ts @@ -1,4 +1,4 @@ -import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths'; +import { PATH_API_VERCEL_BLOB_UPLOAD } from '@/site/paths'; import { copy, del, list } from '@vercel/blob'; import { upload } from '@vercel/blob/client'; @@ -18,7 +18,7 @@ export const vercelBlobUploadFromClient = async ( file, { access: 'public', - handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB, + handleUploadUrl: PATH_API_VERCEL_BLOB_UPLOAD, }, ) .then(({ url }) => url); diff --git a/src/site/paths.ts b/src/site/paths.ts index 77c8d9f2..e6b699ce 100644 --- a/src/site/paths.ts +++ b/src/site/paths.ts @@ -12,6 +12,7 @@ export const PATH_ROOT = '/'; export const PATH_GRID = '/grid'; export const PATH_SETS = '/sets'; export const PATH_ADMIN = '/admin'; +export const PATH_API = '/api'; export const PATH_SIGN_IN = '/sign-in'; export const PATH_OG = '/og'; @@ -31,9 +32,13 @@ const PATH_FILM_SIMULATION_DYNAMIC = `${PREFIX_FILM_SIMULATION}/[simulation]`; export const PATH_ADMIN_PHOTOS = `${PATH_ADMIN}/photos`; export const PATH_ADMIN_UPLOADS = `${PATH_ADMIN}/uploads`; export const PATH_ADMIN_TAGS = `${PATH_ADMIN}/tags`; -export const PATH_ADMIN_UPLOAD_BLOB = `${PATH_ADMIN_UPLOADS}/blob`; export const PATH_ADMIN_CONFIGURATION = `${PATH_ADMIN}/configuration`; +// API paths +export const PATH_API_STORAGE = `${PATH_API}/storage`; +export const PATH_API_VERCEL_BLOB_UPLOAD = `${PATH_API_STORAGE}/vercel-blob`; +export const PATH_API_PRESIGNED_URL = `${PATH_API_STORAGE}/presigned-url`; + // Modifiers const SHARE = 'share'; const NEXT = 'next'; @@ -44,7 +49,6 @@ export const PATHS_ADMIN = [ PATH_ADMIN_PHOTOS, PATH_ADMIN_UPLOADS, PATH_ADMIN_TAGS, - PATH_ADMIN_UPLOAD_BLOB, PATH_ADMIN_CONFIGURATION, ]; From 3bd89f62b5629c6d709f8007177a31a113e68f1f Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sat, 20 Jan 2024 23:40:18 -0600 Subject: [PATCH 4/9] Add listing/deleting Cloudflare blobs --- src/admin/BlobUrls.tsx | 4 +- src/photo/server.ts | 8 +-- src/services/storage/aws-s3.ts | 18 +++---- src/services/storage/cloudflare-r2.ts | 29 ++++++++++- src/services/storage/index.ts | 70 +++++++++++++++++++++------ src/services/storage/vercel-blob.ts | 3 ++ 6 files changed, 100 insertions(+), 32 deletions(-) diff --git a/src/admin/BlobUrls.tsx b/src/admin/BlobUrls.tsx index bed92015..da354a48 100644 --- a/src/admin/BlobUrls.tsx +++ b/src/admin/BlobUrls.tsx @@ -2,7 +2,7 @@ import { Fragment } from 'react'; import AdminGrid from './AdminGrid'; import Link from 'next/link'; import ImageTiny from '@/components/ImageTiny'; -import { fileNameForBlobUrl } from '@/services/storage'; +import { fileNameForStorageUrl } from '@/services/storage'; import FormWithConfirm from '@/components/FormWithConfirm'; import { deleteBlobPhotoAction } from '@/photo/actions'; import DeleteButton from './DeleteButton'; @@ -21,7 +21,7 @@ export default function BlobUrls({ {urls.map(url => { const addUploadPath = pathForAdminUploadUrl(url); - const uploadFileName = fileNameForBlobUrl(url); + const uploadFileName = fileNameForStorageUrl(url); return => { const url = decodeURIComponent(blobPath); - const blobId = getIdFromBlobUrl(url); + const blobId = getIdFromStorageUrl(url); - const extension = getExtensionFromBlobUrl(url); + const extension = getExtensionFromStorageUrl(url); const fileBytes = blobPath ? await fetch(url) diff --git a/src/services/storage/aws-s3.ts b/src/services/storage/aws-s3.ts index 4ce72ee5..b7bb8f55 100644 --- a/src/services/storage/aws-s3.ts +++ b/src/services/storage/aws-s3.ts @@ -22,11 +22,11 @@ export const awsS3Client = () => new S3Client({ }, }); +const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`; + export const isUrlFromAwsS3 = (url: string) => url.startsWith(AWS_S3_BASE_URL); -const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`; - export const awsS3PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' }); @@ -49,16 +49,16 @@ export const awsS3Copy = async ( .then(() => urlForKey(fileNameDestination)); }; -export const awsS3Delete = async (Key: string) => { - awsS3Client().send(new DeleteObjectCommand({ - Bucket: AWS_S3_BUCKET, - Key, - })); -}; - export const awsS3List = async (Prefix: string) => awsS3Client().send(new ListObjectsCommand({ Bucket: AWS_S3_BUCKET, Prefix, })) .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []); + +export const awsS3Delete = async (Key: string) => { + awsS3Client().send(new DeleteObjectCommand({ + Bucket: AWS_S3_BUCKET, + Key, + })); +}; diff --git a/src/services/storage/cloudflare-r2.ts b/src/services/storage/cloudflare-r2.ts index 1595e4d1..5be6d9b0 100644 --- a/src/services/storage/cloudflare-r2.ts +++ b/src/services/storage/cloudflare-r2.ts @@ -1,4 +1,9 @@ -import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { + S3Client, + ListObjectsCommand, + PutObjectCommand, + DeleteObjectCommand, +} from '@aws-sdk/client-s3'; const CLOUDFLARE_R2_BUCKET = process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? ''; @@ -28,5 +33,27 @@ export const cloudflareR2Client = () => new S3Client({ }, }); +const urlForKey = (key?: string, isPublic = true) => isPublic + ? `${CLOUDFLARE_R2_BASE_URL_PUBLIC}/${key}` + : `${CLOUDFLARE_R2_BASE_URL_PRIVATE}/${key}`; + +export const isUrlFromCloudflareR2 = (url: string) => + url.startsWith(CLOUDFLARE_R2_BASE_URL_PRIVATE) || + url.startsWith(CLOUDFLARE_R2_BASE_URL_PUBLIC); + export const cloudflareR2PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key }); + +export const cloudflareR2List = async (Prefix: string) => + cloudflareR2Client().send(new ListObjectsCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + Prefix, + })) + .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []); + +export const cloudflareR2Delete = async (Key: string) => { + cloudflareR2Client().send(new DeleteObjectCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + Key, + })); +}; diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 8e7b0810..79d9b73f 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -15,9 +15,16 @@ import { import { STORAGE_PREFERENCE, HAS_AWS_S3_STORAGE, + HAS_VERCEL_BLOB_STORAGE, + HAS_CLOUDFLARE_R2_STORAGE, } from '@/site/config'; import { generateNanoid } from '@/utility/nanoid'; -import { CLOUDFLARE_R2_BASE_URL_PUBLIC } from './cloudflare-r2'; +import { + CLOUDFLARE_R2_BASE_URL_PUBLIC, + cloudflareR2Delete, + cloudflareR2List, + isUrlFromCloudflareR2, +} from './cloudflare-r2'; import { PATH_API_PRESIGNED_URL } from '@/site/paths'; export const generateBlobId = () => generateNanoid(16); @@ -35,7 +42,7 @@ export const labelForStorage = (type: StorageType): string => { } }; -const blobBaseUrlForStorage = (type: StorageType) => { +export const blobBaseUrlForStorage = (type: StorageType) => { switch (type) { case 'vercel-blob': return VERCEL_BLOB_BASE_URL; case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC; @@ -43,7 +50,15 @@ const blobBaseUrlForStorage = (type: StorageType) => { } }; -export const BLOB_BASE_URL = blobBaseUrlForStorage(STORAGE_PREFERENCE); +export const storageTypeFromUrl = (url: string): StorageType => { + if (isUrlFromCloudflareR2(url)) { + return 'cloudflare-r2'; + } else if (isUrlFromAwsS3(url)) { + return 'aws-s3'; + } else { + return 'vercel-blob'; + } +}; const PREFIX_UPLOAD = 'upload'; const PREFIX_PHOTO = 'photo'; @@ -58,13 +73,21 @@ const REGEX_UPLOAD_ID = new RegExp( 'i', ); -export const fileNameForBlobUrl = (url: string) => - url.replace(`${BLOB_BASE_URL}/`, ''); +export const fileNameForStorageUrl = (url: string) => { + switch (storageTypeFromUrl(url)) { + case 'vercel-blob': + return url.replace(`${VERCEL_BLOB_BASE_URL}/`, ''); + case 'cloudflare-r2': + return url.replace(`${CLOUDFLARE_R2_BASE_URL_PUBLIC}/`, ''); + case 'aws-s3': + return url.replace(`${AWS_S3_BASE_URL}/`, ''); + } +}; -export const getExtensionFromBlobUrl = (url: string) => +export const getExtensionFromStorageUrl = (url: string) => url.match(/.([a-z]{1,4})$/i)?.[1]; -export const getIdFromBlobUrl = (url: string) => +export const getIdFromStorageUrl = (url: string) => url.match(REGEX_UPLOAD_ID)?.[1]; export const isUploadPathnameValid = (pathname?: string) => @@ -87,7 +110,7 @@ export const uploadFromClientViaPresignedUrl = async ( .then((response) => response.text()); return fetch(url, { method: 'PUT', body: file }) - .then(() => `${BLOB_BASE_URL}/${key}`); + .then(() => `${blobBaseUrlForStorage(STORAGE_PREFERENCE)}/${key}`); }; export const uploadPhotoFromClient = async ( @@ -105,7 +128,7 @@ export const convertUploadToPhoto = async ( photoId?: string, ): Promise => { const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`; - const fileExtension = getExtensionFromBlobUrl(uploadUrl); + const fileExtension = getExtensionFromStorageUrl(uploadUrl); const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`; const useAwsS3 = HAS_AWS_S3_STORAGE && isUrlFromAwsS3(uploadUrl); @@ -123,14 +146,29 @@ export const convertUploadToPhoto = async ( return url; }; -export const deleteBlobUrl = (url: string) => - HAS_AWS_S3_STORAGE && isUrlFromAwsS3(url) - ? awsS3Delete(getFileNameFromBlobUrl(url)) - : vercelBlobDelete(url); +export const deleteBlobUrl = (url: string) => { + switch (storageTypeFromUrl(url)) { + case 'vercel-blob': return vercelBlobDelete(url); + case 'cloudflare-r2': return cloudflareR2Delete(getFileNameFromBlobUrl(url)); + case 'aws-s3': return awsS3Delete(getFileNameFromBlobUrl(url)); + } +}; -export const getBlobUploadUrls = (): Promise => HAS_AWS_S3_STORAGE - ? awsS3List(`${PREFIX_UPLOAD}-`) - : vercelBlobList(`${PREFIX_UPLOAD}-`); +export const getBlobUploadUrls = async (): Promise => { + const urls: string[] = []; + + if (HAS_VERCEL_BLOB_STORAGE) { + urls.push(...await vercelBlobList(`${PREFIX_UPLOAD}-`)); + } + if (HAS_AWS_S3_STORAGE) { + urls.push(...await awsS3List(`${PREFIX_UPLOAD}-`)); + } + if (HAS_CLOUDFLARE_R2_STORAGE) { + urls.push(...await cloudflareR2List(`${PREFIX_UPLOAD}-`)); + } + + return urls; +}; export const getBlobPhotoUrls = (): Promise => HAS_AWS_S3_STORAGE ? awsS3List(`${PREFIX_PHOTO}-`) diff --git a/src/services/storage/vercel-blob.ts b/src/services/storage/vercel-blob.ts index 5a664f6b..a8b5094f 100644 --- a/src/services/storage/vercel-blob.ts +++ b/src/services/storage/vercel-blob.ts @@ -9,6 +9,9 @@ const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match( export const VERCEL_BLOB_BASE_URL = `https://${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`; +export const isUrlFromVercelBlob = (url: string) => + url.startsWith(VERCEL_BLOB_BASE_URL); + export const vercelBlobUploadFromClient = async ( file: File | Blob, fileName: string, From 04dd1baef496c0564e2fb8d654432a1ab79095db Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sat, 20 Jan 2024 23:50:57 -0600 Subject: [PATCH 5/9] Tweak checklist copy --- src/site/SiteChecklistClient.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 9417c5d8..39086660 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -144,7 +144,7 @@ export default function SiteChecklistClient({ Date: Sun, 21 Jan 2024 11:14:12 -0600 Subject: [PATCH 6/9] Implement upload -> photo copy in R2 --- src/admin/{BlobUrls.tsx => StorageUrls.tsx} | 2 +- src/app/admin/layout.tsx | 4 +- src/app/admin/photos/page.tsx | 8 +-- src/app/admin/uploads/page.tsx | 8 +-- src/cache/index.ts | 20 +++--- src/photo/actions.ts | 6 +- src/services/storage/aws-s3.ts | 4 +- src/services/storage/cloudflare-r2.ts | 20 ++++++ src/services/storage/index.ts | 76 ++++++++++++++------- 9 files changed, 100 insertions(+), 48 deletions(-) rename src/admin/{BlobUrls.tsx => StorageUrls.tsx} (98%) diff --git a/src/admin/BlobUrls.tsx b/src/admin/StorageUrls.tsx similarity index 98% rename from src/admin/BlobUrls.tsx rename to src/admin/StorageUrls.tsx index da354a48..c4c1ef55 100644 --- a/src/admin/BlobUrls.tsx +++ b/src/admin/StorageUrls.tsx @@ -10,7 +10,7 @@ import { clsx } from 'clsx/lite'; import { pathForAdminUploadUrl } from '@/site/paths'; import AddButton from './AddButton'; -export default function BlobUrls({ +export default function StorageUrls({ title, urls, }: { diff --git a/src/app/admin/layout.tsx b/src/app/admin/layout.tsx index 072b0d73..95efd54a 100644 --- a/src/app/admin/layout.tsx +++ b/src/app/admin/layout.tsx @@ -1,6 +1,6 @@ import AdminNav from '@/admin/AdminNav'; import { - getBlobUploadUrlsNoStore, + getStorageUploadUrlsNoStore, getPhotosCountIncludingHiddenCached, getUniqueTagsCached, } from '@/cache'; @@ -21,7 +21,7 @@ export default async function AdminLayout({ countTags, ] = await Promise.all([ getPhotosCountIncludingHiddenCached(), - getBlobUploadUrlsNoStore() + getStorageUploadUrlsNoStore() .then(urls => urls.length) .catch(e => { console.error(`Error getting blob upload urls: ${e}`); diff --git a/src/app/admin/photos/page.tsx b/src/app/admin/photos/page.tsx index 2474f3d2..27f53955 100644 --- a/src/app/admin/photos/page.tsx +++ b/src/app/admin/photos/page.tsx @@ -14,7 +14,7 @@ import { import { titleForPhoto } from '@/photo'; import MorePhotos from '@/photo/MorePhotos'; import { - getBlobPhotoUrlsNoStore, + getStoragePhotoUrlsNoStore, getPhotosCached, getPhotosCountIncludingHiddenCached, } from '@/cache'; @@ -26,7 +26,7 @@ import { import AdminGrid from '@/admin/AdminGrid'; import DeleteButton from '@/admin/DeleteButton'; import EditButton from '@/admin/EditButton'; -import BlobUrls from '@/admin/BlobUrls'; +import StorageUrls from '@/admin/StorageUrls'; import { PRO_MODE_ENABLED } from '@/site/config'; import SubmitButtonWithStatus from '@/components/SubmitButtonWithStatus'; import IconGrSync from '@/site/IconGrSync'; @@ -45,7 +45,7 @@ export default async function AdminPhotosPage({ ] = await Promise.all([ getPhotosCached({ includeHidden: true, sortBy: 'createdAt', limit }), getPhotosCountIncludingHiddenCached(), - DEBUG_PHOTO_BLOBS ? getBlobPhotoUrlsNoStore() : [], + DEBUG_PHOTO_BLOBS ? getStoragePhotoUrlsNoStore() : [], ]); const showMorePhotos = count > photos.length; @@ -60,7 +60,7 @@ export default async function AdminPhotosPage({ 'border-b pb-6', 'border-gray-200 dark:border-gray-700', )}> - diff --git a/src/app/admin/uploads/page.tsx b/src/app/admin/uploads/page.tsx index 9e4e281a..8108e6eb 100644 --- a/src/app/admin/uploads/page.tsx +++ b/src/app/admin/uploads/page.tsx @@ -1,12 +1,12 @@ -import BlobUrls from '@/admin/BlobUrls'; -import { getBlobUploadUrlsNoStore } from '@/cache'; +import StorageUrls from '@/admin/StorageUrls'; +import { getStorageUploadUrlsNoStore } from '@/cache'; import SiteGrid from '@/components/SiteGrid'; export default async function AdminUploadsPage() { - const blobUrls = await getBlobUploadUrlsNoStore(); + const storageUrls = await getStorageUploadUrlsNoStore(); return ( } + contentMain={} /> ); } diff --git a/src/cache/index.ts b/src/cache/index.ts index d67f3ade..179bbbbb 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -24,7 +24,7 @@ import { getPhotosNearId, } from '@/services/vercel-postgres'; import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo'; -import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/storage'; +import { getStoragePhotoUrls, getStorageUploadUrls } from '@/services/storage'; import type { Session } from 'next-auth'; import { createCameraKey } from '@/camera'; import { PATHS_ADMIN } from '@/site/paths'; @@ -218,15 +218,17 @@ export const getPhotoNoStore = (...args: Parameters) => { return getPhoto(...args); }; -export const getBlobUploadUrlsNoStore: typeof getBlobUploadUrls = (...args) => { - unstable_noStore(); - return getBlobUploadUrls(...args); -}; +export const getStorageUploadUrlsNoStore: typeof getStorageUploadUrls = + (...args) => { + unstable_noStore(); + return getStorageUploadUrls(...args); + }; -export const getBlobPhotoUrlsNoStore: typeof getBlobPhotoUrls = (...args) => { - unstable_noStore(); - return getBlobPhotoUrls(...args); -}; +export const getStoragePhotoUrlsNoStore: typeof getStoragePhotoUrls = + (...args) => { + unstable_noStore(); + return getStoragePhotoUrls(...args); + }; export const getImageCacheHeadersForAuth = (session: Session | null) => { return { diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 1aa5ef2a..bbbe1865 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -16,7 +16,7 @@ import { import { redirect } from 'next/navigation'; import { convertUploadToPhoto, - deleteBlobUrl, + deleteStorageUrl, } from '@/services/storage'; import { revalidateAdminPaths, @@ -66,7 +66,7 @@ export async function toggleFavoritePhoto(photoId: string) { export async function deletePhotoAction(formData: FormData) { await Promise.all([ - deleteBlobUrl(formData.get('url') as string), + deleteStorageUrl(formData.get('url') as string), sqlDeletePhoto(formData.get('id') as string), ]); @@ -94,7 +94,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) { } export async function deleteBlobPhotoAction(formData: FormData) { - await deleteBlobUrl(formData.get('url') as string); + await deleteStorageUrl(formData.get('url') as string); revalidateAdminPaths(); diff --git a/src/services/storage/aws-s3.ts b/src/services/storage/aws-s3.ts index b7bb8f55..2ec518d5 100644 --- a/src/services/storage/aws-s3.ts +++ b/src/services/storage/aws-s3.ts @@ -5,7 +5,7 @@ import { ListObjectsCommand, PutObjectCommand, } from '@aws-sdk/client-s3'; -import { generateBlobId } from '.'; +import { generateStorageId } from '.'; const AWS_S3_BUCKET = process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? ''; const AWS_S3_REGION = process.env.NEXT_PUBLIC_AWS_S3_REGION ?? ''; @@ -38,7 +38,7 @@ export const awsS3Copy = async ( const name = fileNameSource.split('.')[0]; const extension = fileNameSource.split('.')[1]; const Key = addRandomSuffix - ? `${name}-${generateBlobId()}.${extension}` + ? `${name}-${generateStorageId()}.${extension}` : fileNameDestination; return awsS3Client().send(new CopyObjectCommand({ Bucket: AWS_S3_BUCKET, diff --git a/src/services/storage/cloudflare-r2.ts b/src/services/storage/cloudflare-r2.ts index 5be6d9b0..7866005a 100644 --- a/src/services/storage/cloudflare-r2.ts +++ b/src/services/storage/cloudflare-r2.ts @@ -3,7 +3,9 @@ import { ListObjectsCommand, PutObjectCommand, DeleteObjectCommand, + CopyObjectCommand, } from '@aws-sdk/client-s3'; +import { generateStorageId } from '.'; const CLOUDFLARE_R2_BUCKET = process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? ''; @@ -44,6 +46,24 @@ export const isUrlFromCloudflareR2 = (url: string) => export const cloudflareR2PutObjectCommandForKey = (Key: string) => new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key }); +export const cloudflareR2Copy = async ( + fileNameSource: string, + fileNameDestination: string, + addRandomSuffix?: boolean, +) => { + const name = fileNameSource.split('.')[0]; + const extension = fileNameSource.split('.')[1]; + const Key = addRandomSuffix + ? `${name}-${generateStorageId()}.${extension}` + : fileNameDestination; + return cloudflareR2Client().send(new CopyObjectCommand({ + Bucket: CLOUDFLARE_R2_BUCKET, + CopySource: `${CLOUDFLARE_R2_BUCKET}/${fileNameSource}`, + Key, + })) + .then(() => urlForKey(fileNameDestination)); +}; + export const cloudflareR2List = async (Prefix: string) => cloudflareR2Client().send(new ListObjectsCommand({ Bucket: CLOUDFLARE_R2_BUCKET, diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 79d9b73f..64605928 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -21,13 +21,14 @@ import { import { generateNanoid } from '@/utility/nanoid'; import { CLOUDFLARE_R2_BASE_URL_PUBLIC, + cloudflareR2Copy, cloudflareR2Delete, cloudflareR2List, isUrlFromCloudflareR2, } from './cloudflare-r2'; import { PATH_API_PRESIGNED_URL } from '@/site/paths'; -export const generateBlobId = () => generateNanoid(16); +export const generateStorageId = () => generateNanoid(16); export type StorageType = 'vercel-blob' | @@ -42,7 +43,7 @@ export const labelForStorage = (type: StorageType): string => { } }; -export const blobBaseUrlForStorage = (type: StorageType) => { +export const baseUrlForStorage = (type: StorageType) => { switch (type) { case 'vercel-blob': return VERCEL_BLOB_BASE_URL; case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC; @@ -93,7 +94,7 @@ export const getIdFromStorageUrl = (url: string) => export const isUploadPathnameValid = (pathname?: string) => pathname?.match(REGEX_UPLOAD_PATH); -const getFileNameFromBlobUrl = (url: string) => +const getFileNameFromStorageUrl = (url: string) => (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? ''; export const uploadFromClientViaPresignedUrl = async ( @@ -103,14 +104,14 @@ export const uploadFromClientViaPresignedUrl = async ( addRandomSuffix?: boolean, ) => { const key = addRandomSuffix - ? `${fileName}-${generateBlobId()}.${extension}` + ? `${fileName}-${generateStorageId()}.${extension}` : `${fileName}.${extension}`; const url = await fetch(`${PATH_API_PRESIGNED_URL}/${key}`) .then((response) => response.text()); return fetch(url, { method: 'PUT', body: file }) - .then(() => `${blobBaseUrlForStorage(STORAGE_PREFERENCE)}/${key}`); + .then(() => `${baseUrlForStorage(STORAGE_PREFERENCE)}/${key}`); }; export const uploadPhotoFromClient = async ( @@ -131,45 +132,74 @@ export const convertUploadToPhoto = async ( const fileExtension = getExtensionFromStorageUrl(uploadUrl); const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`; - const useAwsS3 = HAS_AWS_S3_STORAGE && isUrlFromAwsS3(uploadUrl); + const storageType = storageTypeFromUrl(uploadUrl); - const url = await (useAwsS3 - ? awsS3Copy(uploadUrl, photoUrl, photoId === undefined) - : vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined)); + let url: string | undefined; + // Copy file + switch (storageType) { + case 'vercel-blob': + url = await vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined); + break; + case 'cloudflare-r2': + url = await cloudflareR2Copy( + getFileNameFromStorageUrl(uploadUrl), + photoUrl, + photoId === undefined, + ); + break; + case 'aws-s3': + url = await awsS3Copy(uploadUrl, photoUrl, photoId === undefined); + break; + } + + // If successful, delete original file if (url) { - await (useAwsS3 - ? awsS3Delete(getFileNameFromBlobUrl(uploadUrl)) - : vercelBlobDelete(uploadUrl)); + switch (storageType) { + case 'vercel-blob': + await vercelBlobDelete(uploadUrl); + break; + case 'cloudflare-r2': + await cloudflareR2Delete(getFileNameFromStorageUrl(uploadUrl)); + break; + case 'aws-s3': + await awsS3Delete(getFileNameFromStorageUrl(uploadUrl)); + break; + } } return url; }; -export const deleteBlobUrl = (url: string) => { +export const deleteStorageUrl = (url: string) => { switch (storageTypeFromUrl(url)) { - case 'vercel-blob': return vercelBlobDelete(url); - case 'cloudflare-r2': return cloudflareR2Delete(getFileNameFromBlobUrl(url)); - case 'aws-s3': return awsS3Delete(getFileNameFromBlobUrl(url)); + case 'vercel-blob': + return vercelBlobDelete(url); + case 'cloudflare-r2': + return cloudflareR2Delete(getFileNameFromStorageUrl(url)); + case 'aws-s3': + return awsS3Delete(getFileNameFromStorageUrl(url)); } }; -export const getBlobUploadUrls = async (): Promise => { +const getStorageUrlsForPrefix = async (prefix = ''): Promise => { const urls: string[] = []; if (HAS_VERCEL_BLOB_STORAGE) { - urls.push(...await vercelBlobList(`${PREFIX_UPLOAD}-`)); + urls.push(...await vercelBlobList(prefix)); } if (HAS_AWS_S3_STORAGE) { - urls.push(...await awsS3List(`${PREFIX_UPLOAD}-`)); + urls.push(...await awsS3List(prefix)); } if (HAS_CLOUDFLARE_R2_STORAGE) { - urls.push(...await cloudflareR2List(`${PREFIX_UPLOAD}-`)); + urls.push(...await cloudflareR2List(prefix)); } return urls; }; -export const getBlobPhotoUrls = (): Promise => HAS_AWS_S3_STORAGE - ? awsS3List(`${PREFIX_PHOTO}-`) - : vercelBlobList(`${PREFIX_PHOTO}-`); +export const getStorageUploadUrls = () => + getStorageUrlsForPrefix(`${PREFIX_UPLOAD}-`); + +export const getStoragePhotoUrls = () => + getStorageUrlsForPrefix(`${PREFIX_PHOTO}-`); From ce7aa2654207c16184d4c2a4263ee00251443363 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sun, 21 Jan 2024 11:26:23 -0600 Subject: [PATCH 7/9] Tweak README hierarchy --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 27f5dec3..d11f01b8 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,11 @@ Installation - `NEXT_PUBLIC_GRID_ASPECT_RATIO = 1.5` sets aspect ratio for grid tiles (defaults to `1`—setting to `0` removes the constraint) - `NEXT_PUBLIC_OG_TEXT_ALIGNMENT = BOTTOM` keeps OG image text bottom aligned (default is top) -### Configure alternate storage +## Configure alternate storage Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations). If you have multiple adapters, you can set one as preferred by storing "aws-s3," "cloudflare-r2," or "vercel-blob" in `NEXT_PUBLIC_STORAGE_PREFERENCE`. -#### Cloudflare R2 +### Cloudflare R2 1. Setup bucket - [Create R2 bucket](https://developers.cloudflare.com/r2/) with default settings @@ -106,7 +106,7 @@ Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used a - `CLOUDFLARE_R2_ACCESS_KEY` - `CLOUDFLARE_R2_SECRET_ACCESS_KEY` -#### AWS S3 +### AWS S3 1. Setup bucket - [Create S3 bucket](https://s3.console.aws.amazon.com/s3) with "ACLs enabled," and "Block all public access" turned off From 3ea54b2a21752595962ff8d035ebd966ff6b71f2 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sun, 21 Jan 2024 11:40:29 -0600 Subject: [PATCH 8/9] Simplify storage configuration text --- src/app/api/storage/presigned-url/[key]/route.ts | 6 +++--- src/services/storage/index.ts | 8 ++++---- src/site/SiteChecklistClient.tsx | 12 +++++++----- src/site/config.ts | 15 +++++++++++---- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/app/api/storage/presigned-url/[key]/route.ts b/src/app/api/storage/presigned-url/[key]/route.ts index 2b77455d..da506a22 100644 --- a/src/app/api/storage/presigned-url/[key]/route.ts +++ b/src/app/api/storage/presigned-url/[key]/route.ts @@ -7,7 +7,7 @@ import { cloudflareR2Client, cloudflareR2PutObjectCommandForKey, } from '@/services/storage/cloudflare-r2'; -import { STORAGE_PREFERENCE } from '@/site/config'; +import { CURRENT_STORAGE } from '@/site/config'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; export const runtime = 'edge'; @@ -19,10 +19,10 @@ export async function GET( const session = await auth(); if (session?.user && key) { const url = await getSignedUrl( - STORAGE_PREFERENCE === 'cloudflare-r2' + CURRENT_STORAGE === 'cloudflare-r2' ? cloudflareR2Client() : awsS3Client(), - STORAGE_PREFERENCE === 'cloudflare-r2' + CURRENT_STORAGE === 'cloudflare-r2' ? cloudflareR2PutObjectCommandForKey(key) : awsS3PutObjectCommandForKey(key), { expiresIn: 3600 } diff --git a/src/services/storage/index.ts b/src/services/storage/index.ts index 64605928..2b58ebe3 100644 --- a/src/services/storage/index.ts +++ b/src/services/storage/index.ts @@ -13,7 +13,7 @@ import { isUrlFromAwsS3, } from './aws-s3'; import { - STORAGE_PREFERENCE, + CURRENT_STORAGE, HAS_AWS_S3_STORAGE, HAS_VERCEL_BLOB_STORAGE, HAS_CLOUDFLARE_R2_STORAGE, @@ -111,15 +111,15 @@ export const uploadFromClientViaPresignedUrl = async ( .then((response) => response.text()); return fetch(url, { method: 'PUT', body: file }) - .then(() => `${baseUrlForStorage(STORAGE_PREFERENCE)}/${key}`); + .then(() => `${baseUrlForStorage(CURRENT_STORAGE)}/${key}`); }; export const uploadPhotoFromClient = async ( file: File | Blob, extension = 'jpg', ) => ( - STORAGE_PREFERENCE === 'cloudflare-r2' || - STORAGE_PREFERENCE === 'aws-s3' + CURRENT_STORAGE === 'cloudflare-r2' || + CURRENT_STORAGE === 'aws-s3' ) ? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true) : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`); diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 39086660..98a20c6f 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -27,7 +27,8 @@ export default function SiteChecklistClient({ hasVercelBlobStorage, hasCloudflareR2Storage, hasAwsS3Storage, - storagePreference, + hasMultipleStorageProviders, + currentStorage, hasAuth, hasAdminUser, hasTitle, @@ -142,10 +143,11 @@ export default function SiteChecklistClient({ and connect to project diff --git a/src/site/config.ts b/src/site/config.ts index 1cc4b59e..c8f131f1 100644 --- a/src/site/config.ts +++ b/src/site/config.ts @@ -56,9 +56,15 @@ export const HAS_AWS_S3_STORAGE = (process.env.AWS_S3_ACCESS_KEY ?? '').length > 0 && (process.env.AWS_S3_SECRET_ACCESS_KEY ?? '').length > 0; -// Storage preference relies on client-only keys -// so that it's available in the browser when uploading -export const STORAGE_PREFERENCE: StorageType = +export const HAS_MULTIPLE_STORAGE_PROVIDERS = [ + HAS_VERCEL_BLOB_STORAGE, + HAS_CLOUDFLARE_R2_STORAGE, + HAS_AWS_S3_STORAGE, +].filter(Boolean).length > 1; + +// Storage preference requires client-available keys +// so it can be reached in the browser when uploading +export const CURRENT_STORAGE: StorageType = (process.env.NEXT_PUBLIC_STORAGE_PREFERENCE as StorageType | undefined) || ( HAS_CLOUDFLARE_R2_STORAGE_CLIENT ? 'cloudflare-r2' @@ -94,7 +100,8 @@ export const CONFIG_CHECKLIST_STATUS = { HAS_VERCEL_BLOB_STORAGE || HAS_CLOUDFLARE_R2_STORAGE || HAS_AWS_S3_STORAGE, - storagePreference: STORAGE_PREFERENCE, + hasMultipleStorageProviders: HAS_MULTIPLE_STORAGE_PROVIDERS, + currentStorage: CURRENT_STORAGE, hasAuth: (process.env.AUTH_SECRET ?? '').length > 0, hasAdminUser: ( (process.env.ADMIN_EMAIL ?? '').length > 0 && From 311b63586b37877f6a9861691e2b75115f04d4cd Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Sun, 21 Jan 2024 11:41:05 -0600 Subject: [PATCH 9/9] Reorder checklist props --- src/site/SiteChecklistClient.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 98a20c6f..6ea9004e 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -40,8 +40,8 @@ export default function SiteChecklistClient({ isPriorityOrderEnabled, isPublicApiEnabled, isOgTextBottomAligned, - showRefreshButton, gridAspectRatio, + showRefreshButton, secret, }: ConfigChecklistStatus & { showRefreshButton?: boolean