Add support for Cloudflare R2 storage

2024-01-20 22:13:05 -06:00 · 2024-01-20 22:13:05 -06:00 · 16c524abc4
commit 16c524abc4
parent 01549ffc88
16 changed files with 208 additions and 55 deletions
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -6,6 +6,7 @@
    "ARROWRIGHT",
    "Astia",
    "camelcase",
+    "cloudflarestorage",
    "CredentialsSignin",
    "Eterna",
    "exif",
--- a/README.md
+++ b/README.md
@ -71,7 +71,42 @@ Installation
 - `NEXT_PUBLIC_GRID_ASPECT_RATIO = 1.5` sets aspect ratio for grid tiles (defaults to `1`—setting to `0` removes the constraint)
 - `NEXT_PUBLIC_OG_TEXT_ALIGNMENT = BOTTOM` keeps OG image text bottom aligned (default is top)

-### Setup alternate storage
+### Configure alternate storage
+
+Only one storage adapter—Vercel Blob, Cloudflare R2, or AWS S3—can be used at a time. Ideally, this is configured before photos are uploaded (see [Issue #34](https://github.com/sambecker/exif-photo-blog/issues/34) for migration considerations).
+
+If you have multiple adapters, you can set one as preferred by storing `aws-s3`, `cloudflare-r2`, or `vercel-blob` in `NEXT_PUBLIC_STORAGE_PREFERENCE`.
+
+#### Cloudflare R2
+
+1. Setup bucket
+   - [Create R2 bucket](https://developers.cloudflare.com/r2/) with default settings
+   - Setup CORS under bucket settings:
+   ```json
+   [{
+       "AllowedHeaders": ["*"]
+       "AllowedOrigins": [
+          "http://localhost:3000",
+          "https://{VERCEL_PROJECT_NAME}*.vercel.app",
+          "{PRODUCTION_DOMAIN}"
+       ],
+       "AllowedMethods": [
+         "GET",
+         "PUT"
+       ],
+   }]
+   ```
+   - Enable R2.dev subdomain (necessary in order to serve files publicly without a custom domain)
+   - Store configuration:
+     - `NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET`: bucket name
+     - `NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID`: account id (found on R2 overview page)
+     - `NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN`: r2.dev subdomain, e.g., "pub-jf90908..."
+2. Setup credentials
+   - Create API token by selecting "Manage R2 API Tokens," and clicking "Create API Token"
+   - Select "Object Read & Write," choose "Apply to specific buckets only," and select the bucket created in Step 1.
+   - Store credentials (⚠️ _Ensure access keys are not prefixed with `NEXT_PUBLIC`_):
+     - `CLOUDFLARE_R2_ACCESS_KEY`
+     - `CLOUDFLARE_R2_SECRET_ACCESS_KEY`

 #### AWS S3

--- a/next.config.js
+++ b/next.config.js
@ -6,6 +6,11 @@ const VERCEL_BLOB_HOSTNAME = VERCEL_BLOB_STORE_ID
  ? `${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`
  : undefined;

+const CLOUDFLARE_R2_HOSTNAME =
+  process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN
+    ? `${process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev`
+    : undefined;
+
 const AWS_S3_HOSTNAME =
  process.env.NEXT_PUBLIC_AWS_S3_BUCKET &&
  process.env.NEXT_PUBLIC_AWS_S3_REGION
@ -28,6 +33,7 @@ const nextConfig = {
    imageSizes: [200],
    remotePatterns: []
      .concat(createRemotePattern(VERCEL_BLOB_HOSTNAME))
+      .concat(createRemotePattern(CLOUDFLARE_R2_HOSTNAME))
      .concat(createRemotePattern(AWS_S3_HOSTNAME)),
    minimumCacheTTL: 31536000,
  },
--- a/src/admin/BlobUrls.tsx
+++ b/src/admin/BlobUrls.tsx
@ -2,7 +2,7 @@ import { Fragment } from 'react';
 import AdminGrid from './AdminGrid';
 import Link from 'next/link';
 import ImageTiny from '@/components/ImageTiny';
-import { fileNameForBlobUrl } from '@/services/blob';
+import { fileNameForBlobUrl } from '@/services/storage';
 import FormWithConfirm from '@/components/FormWithConfirm';
 import { deleteBlobPhotoAction } from '@/photo/actions';
 import DeleteButton from './DeleteButton';
--- a/src/app/admin/uploads/blob/route.tsx
+++ b/src/app/admin/uploads/blob/route.tsx
@ -4,7 +4,7 @@ import {
  ACCEPTED_PHOTO_FILE_TYPES,
  MAX_PHOTO_UPLOAD_SIZE_IN_BYTES,
 } from '@/photo';
-import { isUploadPathnameValid } from '@/services/blob';
+import { isUploadPathnameValid } from '@/services/storage';
 import { handleUpload, type HandleUploadBody } from '@vercel/blob/client';
 import { NextResponse } from 'next/server';

--- a/src/app/api/storage/presigned-url/[key]/route.ts
+++ b/src/app/api/storage/presigned-url/[key]/route.ts
@ -2,7 +2,12 @@ import { auth } from '@/auth';
 import {
  awsS3Client,
  awsS3PutObjectCommandForKey,
-} from '@/services/blob/aws-s3';
+} from '@/services/storage/aws-s3';
+import {
+  cloudflareR2Client,
+  cloudflareR2PutObjectCommandForKey,
+} from '@/services/storage/cloudflare-r2';
+import { STORAGE_PREFERENCE } from '@/site/config';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';

 export const runtime = 'edge';
@ -14,8 +19,12 @@ export async function GET(
  const session = await auth();
  if (session?.user && key) {
    const url = await getSignedUrl(
-      awsS3Client(),
-      awsS3PutObjectCommandForKey(key),
+      STORAGE_PREFERENCE === 'cloudflare-r2'
+        ? cloudflareR2Client()
+        : awsS3Client(),
+      STORAGE_PREFERENCE === 'cloudflare-r2'
+        ? cloudflareR2PutObjectCommandForKey(key)
+        : awsS3PutObjectCommandForKey(key),
      { expiresIn: 3600 }
    );
    return new Response(
--- a/src/cache/index.ts
+++ b/src/cache/index.ts
@ -24,7 +24,7 @@ import {
  getPhotosNearId,
 } from '@/services/vercel-postgres';
 import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo';
-import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob';
+import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/storage';
 import type { Session } from 'next-auth';
 import { createCameraKey } from '@/camera';
 import { PATHS_ADMIN } from '@/site/paths';
--- a/src/photo/PhotoUpload.tsx
+++ b/src/photo/PhotoUpload.tsx
@ -1,7 +1,7 @@
 'use client';

 import { useState } from 'react';
-import { uploadPhotoFromClient } from '@/services/blob';
+import { uploadPhotoFromClient } from '@/services/storage';
 import { useRouter } from 'next/navigation';
 import { PATH_ADMIN_UPLOADS, pathForAdminUploadUrl } from '@/site/paths';
 import ImageInput from '../components/ImageInput';
--- a/src/photo/actions.ts
+++ b/src/photo/actions.ts
@ -17,7 +17,7 @@ import { redirect } from 'next/navigation';
 import {
  convertUploadToPhoto,
  deleteBlobUrl,
-} from '@/services/blob';
+} from '@/services/storage';
 import {
  revalidateAdminPaths,
  revalidateAllKeysAndPaths,
--- a/src/photo/server.ts
+++ b/src/photo/server.ts
@ -1,7 +1,7 @@
 import {
  getExtensionFromBlobUrl,
  getIdFromBlobUrl,
-} from '@/services/blob';
+} from '@/services/storage';
 import { convertExifToFormData } from '@/photo/form';
 import {
  getFujifilmSimulationFromMakerNote,
--- a/src/services/storage/aws-s3.ts
+++ b/src/services/storage/aws-s3.ts
@ -1,4 +1,3 @@
-import { generateNanoid } from '@/utility/nanoid';
 import {
  S3Client,
  CopyObjectCommand,
@ -6,13 +5,14 @@ import {
  ListObjectsCommand,
  PutObjectCommand,
 } from '@aws-sdk/client-s3';
+import { generateBlobId } from '.';

 const AWS_S3_BUCKET = process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? '';
 const AWS_S3_REGION = process.env.NEXT_PUBLIC_AWS_S3_REGION ?? '';
 const AWS_S3_ACCESS_KEY = process.env.AWS_S3_ACCESS_KEY ?? '';
 const AWS_S3_SECRET_ACCESS_KEY = process.env.AWS_S3_SECRET_ACCESS_KEY ?? '';
-
-const API_PATH_PRESIGNED_URL = '/api/aws-s3/presigned-url';
+export const AWS_S3_BASE_URL =
+  `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`;

 export const awsS3Client = () => new S3Client({
  region: AWS_S3_REGION,
@ -22,36 +22,14 @@ export const awsS3Client = () => new S3Client({
  },
 });

-export const AWS_S3_BASE_URL =
-  `https://${AWS_S3_BUCKET}.s3.${AWS_S3_REGION}.amazonaws.com`;
-
 export const isUrlFromAwsS3 = (url: string) =>
  url.startsWith(AWS_S3_BASE_URL);

 const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`;

-const generateBlobId = () => generateNanoid(16);
-
 export const awsS3PutObjectCommandForKey = (Key: string) =>
  new PutObjectCommand({ Bucket: AWS_S3_BUCKET, Key, ACL: 'public-read' });

-export const awsS3UploadFromClient = async (
-  file: File | Blob,
-  fileName: string,
-  extension: string,
-  addRandomSuffix?: boolean,
-) => {
-  const key = addRandomSuffix
-    ? `${fileName}-${generateBlobId()}.${extension}`
-    : `${fileName}.${extension}`;
-
-  const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`)
-    .then((response) => response.text());
-
-  return fetch(url, { method: 'PUT', body: file })
-    .then(() => urlForKey(key));
-};
-
 export const awsS3Copy = async (
  fileNameSource: string,
  fileNameDestination: string,
--- a/src/services/storage/cloudflare-r2.ts
+++ b/src/services/storage/cloudflare-r2.ts
@ -0,0 +1,32 @@
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+
+const CLOUDFLARE_R2_BUCKET =
+  process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? '';
+const CLOUDFLARE_R2_ACCOUNT_ID =
+  process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? '';
+const CLOUDFLARE_R2_DEV_SUBDOMAIN =
+  process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? '';
+const CLOUDFLARE_R2_ACCESS_KEY =
+  process.env.CLOUDFLARE_R2_ACCESS_KEY ?? '';
+const CLOUDFLARE_R2_SECRET_ACCESS_KEY =
+  process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? '';
+const CLOUDFLARE_R2_ENDPOINT =
+  `https://${CLOUDFLARE_R2_ACCOUNT_ID}.r2.cloudflarestorage.com`;
+
+export const CLOUDFLARE_R2_BASE_URL_PRIVATE =
+  `${CLOUDFLARE_R2_ENDPOINT}/${CLOUDFLARE_R2_BUCKET}`;
+
+export const CLOUDFLARE_R2_BASE_URL_PUBLIC =
+  `https://${CLOUDFLARE_R2_DEV_SUBDOMAIN}.r2.dev`;
+
+export const cloudflareR2Client = () => new S3Client({
+  region: 'auto',
+  endpoint: CLOUDFLARE_R2_ENDPOINT,
+  credentials: {
+    accessKeyId: CLOUDFLARE_R2_ACCESS_KEY,
+    secretAccessKey: CLOUDFLARE_R2_SECRET_ACCESS_KEY,
+  },
+});
+
+export const cloudflareR2PutObjectCommandForKey = (Key: string) =>
+  new PutObjectCommand({ Bucket: CLOUDFLARE_R2_BUCKET, Key });
--- a/src/services/storage/index.ts
+++ b/src/services/storage/index.ts
@ -10,16 +10,44 @@ import {
  awsS3Copy,
  awsS3Delete,
  awsS3List,
-  awsS3UploadFromClient,
  isUrlFromAwsS3,
 } from './aws-s3';
-import { HAS_AWS_S3_STORAGE, HAS_AWS_S3_STORAGE_CLIENT } from '@/site/config';
+import {
+  STORAGE_PREFERENCE,
+  HAS_AWS_S3_STORAGE,
+} from '@/site/config';
+import { generateNanoid } from '@/utility/nanoid';
+import { CLOUDFLARE_R2_BASE_URL_PUBLIC } from './cloudflare-r2';
+
+export const generateBlobId = () => generateNanoid(16);
+
+export type StorageType =
+  'vercel-blob' |
+  'aws-s3' |
+  'cloudflare-r2';
+
+export const labelForStorage = (type: StorageType): string => {
+  switch (type) {
+  case 'vercel-blob': return 'Vercel Blob';
+  case 'cloudflare-r2': return 'Cloudflare R2';
+  case 'aws-s3': return 'AWS S3';
+  }
+};
+
+const blobBaseUrlForStorage = (type: StorageType) => {
+  switch (type) {
+  case 'vercel-blob': return VERCEL_BLOB_BASE_URL;
+  case 'cloudflare-r2': return CLOUDFLARE_R2_BASE_URL_PUBLIC;
+  case 'aws-s3': return AWS_S3_BASE_URL;
+  }
+};
+
+export const BLOB_BASE_URL = blobBaseUrlForStorage(STORAGE_PREFERENCE);
+
+const API_PATH_PRESIGNED_URL = '/api/storage/presigned-url';

 const PREFIX_UPLOAD = 'upload';
 const PREFIX_PHOTO = 'photo';
-const BLOB_BASE_URL = HAS_AWS_S3_STORAGE_CLIENT
-  ? AWS_S3_BASE_URL
-  : VERCEL_BLOB_BASE_URL;

 const REGEX_UPLOAD_PATH = new RegExp(
  `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
@ -46,11 +74,31 @@ export const isUploadPathnameValid = (pathname?: string) =>
 const getFileNameFromBlobUrl = (url: string) =>
  (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? '';

+export const uploadFromClientViaPresignedUrl = async (
+  file: File | Blob,
+  fileName: string,
+  extension: string,
+  addRandomSuffix?: boolean,
+) => {
+  const key = addRandomSuffix
+    ? `${fileName}-${generateBlobId()}.${extension}`
+    : `${fileName}.${extension}`;
+
+  const url = await fetch(`${API_PATH_PRESIGNED_URL}/${key}`)
+    .then((response) => response.text());
+
+  return fetch(url, { method: 'PUT', body: file })
+    .then(() => `${BLOB_BASE_URL}/${key}`);
+};
+
 export const uploadPhotoFromClient = async (
  file: File | Blob,
  extension = 'jpg',
-) => HAS_AWS_S3_STORAGE_CLIENT
-  ? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true)
+) => (
+  STORAGE_PREFERENCE === 'cloudflare-r2' ||
+  STORAGE_PREFERENCE === 'aws-s3'
+)
+  ? uploadFromClientViaPresignedUrl(file, PREFIX_UPLOAD, extension, true)
  : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`);

 export const convertUploadToPhoto = async (
--- a/src/services/storage/vercel-blob.ts
+++ b/src/services/storage/vercel-blob.ts
--- a/src/site/SiteChecklistClient.tsx
+++ b/src/site/SiteChecklistClient.tsx
@ -19,12 +19,15 @@ import Checklist from '@/components/Checklist';
 import { toastSuccess } from '@/toast';
 import { ConfigChecklistStatus } from './config';
 import StatusIcon from '@/components/StatusIcon';
+import { labelForStorage } from '@/services/storage';

 export default function SiteChecklistClient({
  hasPostgres,
-  hasBlob,
-  hasVercelBlob,
+  hasStorage,
+  hasVercelBlobStorage,
+  hasCloudflareR2Storage,
  hasAwsS3Storage,
+  storagePreference,
  hasAuth,
  hasAdminUser,
  hasTitle,
@ -139,14 +142,17 @@ export default function SiteChecklistClient({
          and connect to project
        </ChecklistRow>
        <ChecklistRow
-          title="Setup blob store (one of the following)"
-          status={hasBlob}
+          title={hasStorage
+            // eslint-disable-next-line max-len
+            ? `Setup storage (preferred adapter: ${labelForStorage(storagePreference)})`
+            : 'Setup storage (one of the following)'}
+          status={hasStorage}
          isPending={isPendingPage}
        >
          {renderSubStatus(
-            hasVercelBlob ? 'checked' : 'optional',
+            hasVercelBlobStorage ? 'checked' : 'optional',
            <>
-              Vercel Blob:
+              {labelForStorage('vercel-blob')}:
              {' '}
              {renderLink(
                // eslint-disable-next-line max-len
@ -157,10 +163,21 @@ export default function SiteChecklistClient({
              and connect to project
            </>,
          )}
+          {renderSubStatus(
+            hasCloudflareR2Storage ? 'checked' : 'optional',
+            <>
+              {labelForStorage('cloudflare-r2')}:
+              {' '}
+              {renderLink(
+                'https://github.com/sambecker/exif-photo-blog#cloudflare-r2',
+                'create/configure bucket',
+              )}
+            </>
+          )}
          {renderSubStatus(
            hasAwsS3Storage ? 'checked' : 'optional',
            <>
-              AWS S3:
+              {labelForStorage('aws-s3')}:
              {' '}
              {renderLink(
                'https://github.com/sambecker/exif-photo-blog#aws-s3',
--- a/src/site/config.ts
+++ b/src/site/config.ts
@ -1,3 +1,4 @@
+import type { StorageType } from '@/services/storage';
 import { makeUrlAbsolute, shortenUrl } from '@/utility/url';

 // META / DOMAINS
@ -31,12 +32,22 @@ export const BASE_URL = process.env.NODE_ENV === 'production'
  : 'http://localhost:3000';

 // STORAGE: VERCEL BLOB
-export const HAS_VERCEL_BLOB =
+export const HAS_VERCEL_BLOB_STORAGE =
  (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0;

+// STORAGE: Cloudflare R2
+// Includes separate check for client-side usage, i.e., url construction
+export const HAS_CLOUDFLARE_R2_STORAGE_CLIENT =
+  (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_BUCKET ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_ACCOUNT_ID ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_CLOUDFLARE_R2_DEV_SUBDOMAIN ?? '').length > 0;
+export const HAS_CLOUDFLARE_R2_STORAGE =
+  HAS_CLOUDFLARE_R2_STORAGE_CLIENT &&
+  (process.env.CLOUDFLARE_R2_ACCESS_KEY ?? '').length > 0 &&
+  (process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY ?? '').length > 0;
+
 // STORAGE: AWS S3
-// Includes separate check for client-side usage,
-// i.e., uploading, url construction
+// Includes separate check for client-side usage, i.e., url construction
 export const HAS_AWS_S3_STORAGE_CLIENT =
  (process.env.NEXT_PUBLIC_AWS_S3_BUCKET ?? '').length > 0 &&
  (process.env.NEXT_PUBLIC_AWS_S3_REGION ?? '').length > 0;
@ -45,6 +56,17 @@ export const HAS_AWS_S3_STORAGE =
  (process.env.AWS_S3_ACCESS_KEY ?? '').length > 0 &&
  (process.env.AWS_S3_SECRET_ACCESS_KEY ?? '').length > 0;

+// Storage preference relies on client-only keys
+// so that it's available in the browser when uploading
+export const STORAGE_PREFERENCE: StorageType =
+  (process.env.NEXT_PUBLIC_STORAGE_PREFERENCE as StorageType | undefined) || (
+    HAS_CLOUDFLARE_R2_STORAGE_CLIENT
+      ? 'cloudflare-r2'
+      : HAS_AWS_S3_STORAGE_CLIENT
+        ? 'aws-s3'
+        : 'vercel-blob'
+  );
+
 // SETTINGS

 export const PRO_MODE_ENABLED = process.env.NEXT_PUBLIC_PRO_MODE === '1';
@ -65,9 +87,14 @@ export const HIGH_DENSITY_GRID = GRID_ASPECT_RATIO <= 1;

 export const CONFIG_CHECKLIST_STATUS = {
  hasPostgres: (process.env.POSTGRES_HOST ?? '').length > 0,
-  hasBlob: HAS_VERCEL_BLOB || HAS_AWS_S3_STORAGE,
-  hasVercelBlob: HAS_VERCEL_BLOB,
+  hasVercelBlobStorage: HAS_VERCEL_BLOB_STORAGE,
+  hasCloudflareR2Storage: HAS_CLOUDFLARE_R2_STORAGE,
  hasAwsS3Storage: HAS_AWS_S3_STORAGE,
+  hasStorage:
+    HAS_VERCEL_BLOB_STORAGE ||
+    HAS_CLOUDFLARE_R2_STORAGE ||
+    HAS_AWS_S3_STORAGE,
+  storagePreference: STORAGE_PREFERENCE,
  hasAuth: (process.env.AUTH_SECRET ?? '').length > 0,
  hasAdminUser: (
    (process.env.ADMIN_EMAIL ?? '').length > 0 &&
@ -89,6 +116,6 @@ export type ConfigChecklistStatus = typeof CONFIG_CHECKLIST_STATUS;

 export const IS_SITE_READY =
  CONFIG_CHECKLIST_STATUS.hasPostgres &&
-  CONFIG_CHECKLIST_STATUS.hasBlob &&
+  CONFIG_CHECKLIST_STATUS.hasStorage &&
  CONFIG_CHECKLIST_STATUS.hasAuth &&
  CONFIG_CHECKLIST_STATUS.hasAdminUser;