Merge pull request #21 from sambecker/blob-adapter

Blob adapter: S3
2023-11-26 19:57:59 -06:00 · 2023-11-26 19:57:59 -06:00 · be5d389b20
commit be5d389b20
parent f552cf4fc7 4d655eb76a
15 changed files with 1390 additions and 113 deletions
--- a/README.md
+++ b/README.md
@ -68,6 +68,43 @@ Installation
 - `NEXT_PUBLIC_HIDE_REPO_LINK = 1` removes footer link to repo
 - `NEXT_PUBLIC_HIDE_FILM_SIMULATIONS = 1` prevents Fujifilm simulations showing up in `/grid` sidebar

+### Setup alternate storage
+
+#### AWS S3
+
+1. [Create bucket](https://s3.console.aws.amazon.com/s3) with "ACLs enabled," and "Block all public access" turned off
+   - Setup CORS:
+     ```
+     [{
+      "AllowedHeaders": ["*"],
+        "AllowedMethods": [
+         "GET",
+         "PUT"
+        ],
+        "AllowedOrigins": [
+          "http://localhost:*",
+          "https://${VERCEL_PROJECT_NAME}*.vercel.app"
+          "{PRODUCTION_DOMAIN}",
+        ],
+        "ExposeHeaders": []
+     }]
+     ```
+   - Store configuration 
+     - `NEXT_PUBLIC_S3_BUCKET`
+     - `NEXT_PUBLIC_S3_REGION`
+2. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for client uploads (JSON editor recommended)
+   - Action: `s3:PutObject`, `s3:PutObjectACL`
+   - Resource: `arn:aws:s3:::{BUCKET_NAME}/upload-*`
+3. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for admin actions (JSON editor recommended)
+   - Action: `s3:PutObject`, `s3:PutObjectACL`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject`
+   - Resource: `arn:aws:s3:::{BUCKET_NAME}`, `arn:aws:s3:::{BUCKET_NAME}/*`
+4. [Create IAM user](https://console.aws.amazon.com/iam/home#/users) for upload policy (by choosing "Attach policies directly"), create access key under "Security credentials," choose "Application running outside AWS," and store credentials
+   - `NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY`
+   - `NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY`
+5. [Create IAM user](https://console.aws.amazon.com/iam/home#/users), for admin policy (by choosing "Attach policies directly"), , create access key under "Security credentials," choose "Application running outside AWS," and store credentials (_ensure admin environment variables are not prefixed with `NEXT_PUBLIC`_)
+   - `S3_ADMIN_ACCESS_KEY`
+   - `S3_ADMIN_SECRET_ACCESS_KEY`
+
 FAQ
 -
 Q: My images/content have fallen out of sync with my database and/or my production site no longer matches local development. What do I do?<br />
--- a/next.config.js
+++ b/next.config.js
@ -1,24 +1,40 @@
-/** @type {import('next').NextConfig} */
-
-const STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
+const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
  /^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
 )?.[1].toLowerCase();
+const VERCEL_BLOB_HOSTNAME = VERCEL_BLOB_STORE_ID
+  ? `${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`
+  : undefined;
+
+const AWS_S3_HOSTNAME =
+  process.env.NEXT_PUBLIC_S3_BUCKET &&
+  process.env.NEXT_PUBLIC_S3_REGION
+    // eslint-disable-next-line max-len
+    ? `${process.env.NEXT_PUBLIC_S3_BUCKET}.s3.${process.env.NEXT_PUBLIC_S3_REGION}.amazonaws.com`
+    : undefined;
+
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  images: {
+    imageSizes: [200],
+    remotePatterns: []
+      .concat(VERCEL_BLOB_HOSTNAME ? {
+        protocol: 'https',
+        hostname: VERCEL_BLOB_HOSTNAME,
+        port: '',
+        pathname: '/**',
+      } : [])
+      .concat(AWS_S3_HOSTNAME ? {
+        protocol: 'https',
+        hostname: AWS_S3_HOSTNAME,
+        port: '',
+        pathname: '/**',
+      } : []),
+    minimumCacheTTL: 31536000,
+  },
+};

 const withBundleAnalyzer = require('@next/bundle-analyzer')({
  enabled: process.env.ANALYZE === 'true',
 });

-const nextConfig = {
-  images: {
-    imageSizes: [200],
-    remotePatterns: [{
-      protocol: 'https',
-      hostname: `${STORE_ID}.public.blob.vercel-storage.com`,
-      port: '',
-      pathname: '/**',
-    }],
-    minimumCacheTTL: 31536000,
-  },
-};
-
 module.exports = withBundleAnalyzer(nextConfig);
--- a/package.json
+++ b/package.json
@ -9,6 +9,7 @@
    "analyze": "ANALYZE=true next build"
  },
  "dependencies": {
+    "@aws-sdk/client-s3": "^3.456.0",
    "@next/bundle-analyzer": "14.0.3",
    "@tailwindcss/forms": "^0.5.7",
    "@testing-library/jest-dom": "^6.1.4",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/src/cache/index.ts
+++ b/src/cache/index.ts
@ -20,7 +20,7 @@ import {
  getUniqueFilmSimulations,
  getPhotosFilmSimulationDateRange,
  getPhotosFilmSimulationCount,
-} from '@/services/postgres';
+} from '@/services/vercel-postgres';
 import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo';
 import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob';
 import type { Session } from 'next-auth';
--- a/src/photo/PhotoUpload.tsx
+++ b/src/photo/PhotoUpload.tsx
@ -56,7 +56,7 @@ export default function PhotoUpload({
                  blob,
                  extension,
                )
-                  .then(({ url }) => {
+                  .then(url => {
                    if (isLastBlob) {
                      // Refresh page to update upload list,
                      // relevant to upload count in nav
--- a/src/photo/actions.ts
+++ b/src/photo/actions.ts
@ -7,7 +7,7 @@ import {
  sqlUpdatePhoto,
  sqlRenamePhotoTagGlobally,
  getPhoto,
-} from '@/services/postgres';
+} from '@/services/vercel-postgres';
 import {
  PhotoFormData,
  convertFormDataToPhotoDbInsert,
@ -16,7 +16,7 @@ import {
 import { redirect } from 'next/navigation';
 import {
  convertUploadToPhoto,
-  deleteBlobPhoto,
+  deleteBlobUrl,
 } from '@/services/blob';
 import {
  revalidateAdminPaths,
@ -52,7 +52,7 @@ export async function updatePhotoAction(formData: FormData) {

 export async function deletePhotoAction(formData: FormData) {
  await Promise.all([
-    deleteBlobPhoto(formData.get('url') as string),
+    deleteBlobUrl(formData.get('url') as string),
    sqlDeletePhoto(formData.get('id') as string),
  ]);

@ -80,7 +80,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) {
 }

 export async function deleteBlobPhotoAction(formData: FormData) {
-  await deleteBlobPhoto(formData.get('url') as string);
+  await deleteBlobUrl(formData.get('url') as string);

  revalidateAdminPaths();

--- a/src/photo/server.ts
+++ b/src/photo/server.ts
@ -1,4 +1,7 @@
-import { getExtensionFromBlobUrl, getIdFromBlobUrl } from '@/services/blob';
+import {
+  getExtensionFromBlobUrl,
+  getIdFromBlobUrl,
+} from '@/services/blob';
 import { convertExifToFormData } from '@/photo/form';
 import {
  getFujifilmSimulationFromMakerNote,
--- a/src/services/blob.ts
+++ b/src/services/blob.ts
@ -1,82 +0,0 @@
-import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
-import { copy, del, list } from '@vercel/blob';
-import { upload } from '@vercel/blob/client';
-
-const STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
-  /^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
-)?.[1].toLowerCase();
-
-export const BLOB_BASE_URL =
-  `https://${STORE_ID}.public.blob.vercel-storage.com`;
-
-const PREFIX_UPLOAD = 'upload';
-const PREFIX_PHOTO = 'photo';
-
-const REGEX_UPLOAD_PATH = new RegExp(
-  `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
-  'i',
-);
-
-const REGEX_UPLOAD_ID = new RegExp(
-  `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
-  'i',
-);
-
-export const pathForBlobUrl = (url: string) =>
-  url.replace(`${BLOB_BASE_URL}/`, '');
-
-export const getExtensionFromBlobUrl = (url: string) =>
-  url.match(/.([a-z]{1,4})$/i)?.[1];
-
-export const getIdFromBlobUrl = (url: string) =>
-  url.match(REGEX_UPLOAD_ID)?.[1];
-
-export const isUploadPathnameValid = (pathname?: string) =>
-  pathname?.match(REGEX_UPLOAD_PATH);
-
-export const uploadPhotoFromClient = async (
-  file: File | Blob,
-  extension = 'jpg',
-) =>
-  upload(
-    `${PREFIX_UPLOAD}.${extension}`,
-    file,
-    {
-      access: 'public',
-      handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
-    },
-  );
-
-export const convertUploadToPhoto = async (
-  uploadUrl: string,
-  photoId?: string,
-) => {
-  const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
-  const fileExtension = getExtensionFromBlobUrl(uploadUrl) ?? 'jpg';
-  const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
-
-  const { url } = await copy(
-    uploadUrl,
-    photoUrl,
-    {
-      access: 'public',
-      ...photoId && { addRandomSuffix: false },
-    }
-  );
-
-  if (url) {
-    await del(uploadUrl);
-  }
-
-  return url;
-};
-
-export const deleteBlobPhoto = (url: string) => del(url);
-
-export const getBlobUploadUrls = () =>
-  list({ prefix: `${PREFIX_UPLOAD}-` })
-    .then(({ blobs }) => blobs.map(({ url }) => url));
-
-export const getBlobPhotoUrls = () =>
-  list({ prefix: `${PREFIX_PHOTO}-` })
-    .then(({ blobs }) => blobs.map(({ url }) => url));
--- a/src/services/blob/aws-s3.ts
+++ b/src/services/blob/aws-s3.ts
@ -0,0 +1,95 @@
+import { generateNanoid } from '@/utility/nanoid';
+import {
+  S3Client,
+  CopyObjectCommand,
+  DeleteObjectCommand,
+  ListObjectsCommand,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3';
+
+const S3_BUCKET = process.env.NEXT_PUBLIC_S3_BUCKET ?? '';
+const S3_REGION = process.env.NEXT_PUBLIC_S3_REGION ?? '';
+const S3_UPLOAD_ACCESS_KEY =
+  process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '';
+const S3_UPLOAD_SECRET_ACCESS_KEY =
+  process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '';
+const S3_ADMIN_ACCESS_KEY =
+  process.env.S3_ADMIN_ACCESS_KEY;
+const S3_ADMIN_SECRET_ACCESS_KEY =
+  process.env.S3_ADMIN_SECRET_ACCESS_KEY;
+
+export const HAS_AWS_S3_STORAGE =
+  S3_BUCKET.length > 0 &&
+  S3_REGION.length > 0 &&
+  S3_UPLOAD_ACCESS_KEY.length > 0 &&
+  S3_UPLOAD_SECRET_ACCESS_KEY.length > 0;
+
+const client = () => new S3Client({
+  region: S3_REGION,
+  credentials: {
+    // Fall back on upload credentials if admin credentials are not available
+    accessKeyId: S3_ADMIN_ACCESS_KEY ?? S3_UPLOAD_ACCESS_KEY,
+    secretAccessKey: S3_ADMIN_SECRET_ACCESS_KEY ?? S3_UPLOAD_SECRET_ACCESS_KEY,
+  },
+});
+
+export const AWS_S3_BASE_URL =
+  `https://${S3_BUCKET}.s3.${S3_REGION}.amazonaws.com`;
+
+export const isUrlFromAwsS3 = (url: string) =>
+  url.startsWith(AWS_S3_BASE_URL);
+
+const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`;
+
+const generateBlobId = () => generateNanoid(16);
+
+export const awsS3UploadFromClient = async (
+  file: File | Blob,
+  fileName: string,
+  extension: string,
+  addRandomSuffix?: boolean,
+) => {
+  const Key = addRandomSuffix
+    ? `${fileName}-${generateBlobId()}.${extension}`
+    : `${fileName}.${extension}`;
+  return client().send(new PutObjectCommand({
+    Bucket: S3_BUCKET,
+    Key,
+    Body: file,
+    ACL: 'public-read',
+  }))
+    .then(() => urlForKey(Key));
+};
+
+export const awsS3Copy = async (
+  fileNameSource: string,
+  fileNameDestination: string,
+  addRandomSuffix?: boolean,
+) => {
+  const name = fileNameSource.split('.')[0];
+  const extension = fileNameSource.split('.')[1];
+  const Key = addRandomSuffix
+    ? `${name}-${generateBlobId()}.${extension}`
+    : fileNameDestination;
+  return client().send(new CopyObjectCommand({
+    Bucket: S3_BUCKET,
+    CopySource: fileNameSource,
+    Key,
+    ACL: 'public-read',
+  }))
+    .then(() => urlForKey(fileNameDestination));
+};
+
+export const awsS3Delete = async (Key: string) => {
+  client().send(new DeleteObjectCommand({
+    Bucket: S3_BUCKET,
+    Key,
+  }));
+};
+
+export const awsS3List = async (Prefix: string) =>
+  client().send(new ListObjectsCommand({
+    Bucket: S3_BUCKET,
+    Prefix,
+  }))
+    .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []);
--- a/src/services/blob/index.ts
+++ b/src/services/blob/index.ts
@ -0,0 +1,86 @@
+import {
+  VERCEL_BLOB_BASE_URL,
+  vercelBlobCopy,
+  vercelBlobDelete,
+  vercelBlobList,
+  vercelBlobUploadFromClient,
+} from './vercel-blob';
+import {
+  AWS_S3_BASE_URL,
+  HAS_AWS_S3_STORAGE,
+  awsS3Copy,
+  awsS3Delete,
+  awsS3List,
+  awsS3UploadFromClient,
+  isUrlFromAwsS3,
+} from './aws-s3';
+
+const PREFIX_UPLOAD = 'upload';
+const PREFIX_PHOTO = 'photo';
+const BLOB_BASE_URL = AWS_S3_BASE_URL ?? VERCEL_BLOB_BASE_URL;
+
+const REGEX_UPLOAD_PATH = new RegExp(
+  `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
+  'i',
+);
+
+const REGEX_UPLOAD_ID = new RegExp(
+  `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
+  'i',
+);
+
+export const pathForBlobUrl = (url: string) =>
+  url.replace(`${BLOB_BASE_URL}/`, '');
+
+export const getExtensionFromBlobUrl = (url: string) =>
+  url.match(/.([a-z]{1,4})$/i)?.[1];
+
+export const getIdFromBlobUrl = (url: string) =>
+  url.match(REGEX_UPLOAD_ID)?.[1];
+
+export const isUploadPathnameValid = (pathname?: string) =>
+  pathname?.match(REGEX_UPLOAD_PATH);
+
+const getFileNameFromBlobUrl = (url: string) =>
+  (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? '';
+
+export const uploadPhotoFromClient = async (
+  file: File | Blob,
+  extension = 'jpg',
+) => HAS_AWS_S3_STORAGE
+  ? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true)
+  : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`);
+
+export const convertUploadToPhoto = async (
+  uploadUrl: string,
+  photoId?: string,
+): Promise<string> => {
+  const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
+  const fileExtension = getExtensionFromBlobUrl(uploadUrl);
+  const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
+
+  const url = await (HAS_AWS_S3_STORAGE
+    ? awsS3Copy(uploadUrl, photoUrl, photoId === undefined)
+    : vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined));
+
+  if (url) {
+    await (HAS_AWS_S3_STORAGE
+      ? awsS3Delete(getFileNameFromBlobUrl(uploadUrl))
+      : vercelBlobDelete(uploadUrl));
+  }
+
+  return url;
+};
+
+export const deleteBlobUrl = (url: string) =>
+  HAS_AWS_S3_STORAGE && isUrlFromAwsS3(url)
+    ? awsS3Delete(getFileNameFromBlobUrl(url))
+    : vercelBlobDelete(url);
+
+export const getBlobUploadUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
+  ? awsS3List(`${PREFIX_UPLOAD}-`)
+  : vercelBlobList(`${PREFIX_UPLOAD}-`);
+
+export const getBlobPhotoUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
+  ? awsS3List(`${PREFIX_PHOTO}-`)
+  : vercelBlobList(`${PREFIX_PHOTO}-`);
--- a/src/services/blob/vercel-blob.ts
+++ b/src/services/blob/vercel-blob.ts
@ -0,0 +1,44 @@
+import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
+import { copy, del, list } from '@vercel/blob';
+import { upload } from '@vercel/blob/client';
+
+const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
+  /^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
+)?.[1].toLowerCase();
+
+export const VERCEL_BLOB_BASE_URL =
+  `https://${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`;
+
+export const vercelBlobUploadFromClient = async (
+  file: File | Blob,
+  fileName: string,
+) =>
+  upload(
+    fileName,
+    file,
+    {
+      access: 'public',
+      handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
+    },
+  )
+    .then(({ url }) => url);
+
+export const vercelBlobCopy = (
+  fileNameSource: string,
+  fileNameDestination: string,
+  addRandomSuffix?: boolean,
+): Promise<string> =>
+  copy(
+    fileNameSource,
+    fileNameDestination,
+    {
+      access: 'public',
+      addRandomSuffix,
+    },
+  )
+    .then(({ url }) => url);
+
+export const vercelBlobDelete = (fileName: string) => del(fileName);
+
+export const vercelBlobList = (prefix: string) => list({ prefix })
+  .then(({ blobs }) => blobs.map(({ url }) => url));
--- a/src/services/vercel-postgres.ts
+++ b/src/services/vercel-postgres.ts
--- a/src/site/SiteChecklistClient.tsx
+++ b/src/site/SiteChecklistClient.tsx
@ -121,16 +121,30 @@ export default function SiteChecklistClient({
          and connect to project
        </ChecklistRow>
        <ChecklistRow
-          title="Setup blob store"
+          title="Setup blob store (one of the following)"
          status={hasBlob}
          isPending={isPendingPage}
        >
-          {renderLink(
-            'https://vercel.com/docs/storage/vercel-blob/quickstart',
-            'Create Vercel Blob store',
-          )}
-          {' '}
-          and connect to project
+          <ol className="list-decimal list-inside">
+            <li>
+              Vercel Blob:
+              {' '}
+              {renderLink(
+                'https://vercel.com/docs/storage/vercel-blob/quickstart',
+                'create store',
+              )}
+              {' '} 
+              and connect to project
+            </li>
+            <li>
+              AWS S3:
+              {' '}
+              {renderLink(
+                'https://github.com/sambecker/exif-photo-blog#aws-s3',
+                'create/configure bucket',
+              )}
+            </li>
+          </ol>
        </ChecklistRow>
      </Checklist>
      <Checklist
--- a/src/site/config.ts
+++ b/src/site/config.ts
@ -1,5 +1,7 @@
 import { makeUrlAbsolute, shortenUrl } from '@/utility/url';

+// META / DOMAINS
+
 export const SITE_TITLE =
  process.env.NEXT_PUBLIC_SITE_TITLE ||
  'Photo Blog';
@ -28,6 +30,20 @@ export const BASE_URL = process.env.NODE_ENV === 'production'
  ? makeUrlAbsolute(SITE_DOMAIN).toLowerCase()
  : 'http://localhost:3000';

+// STORAGE
+
+const hasVercelBlob = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0;
+const hasAwsS3Storage =
+  (process.env.NEXT_PUBLIC_S3_BUCKET ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_S3_REGION ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '').length > 0 &&
+  (process.env.S3_ADMIN_ACCESS_KEY ?? '').length > 0 &&
+  (process.env.S3_ADMIN_SECRET_ACCESS_KEY ?? '').length > 0;
+  
+
+// SETTINGS
+
 export const PRO_MODE_ENABLED = process.env.NEXT_PUBLIC_PRO_MODE === '1';
 export const PUBLIC_API_ENABLED = process.env.NEXT_PUBLIC_PUBLIC_API === '1';
 export const SHOW_REPO_LINK = process.env.NEXT_PUBLIC_HIDE_REPO_LINK !== '1';
@ -38,7 +54,7 @@ export const OG_TEXT_BOTTOM_ALIGNMENT =

 export const CONFIG_CHECKLIST_STATUS = {
  hasPostgres: (process.env.POSTGRES_HOST ?? '').length > 0,
-  hasBlob: (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0,
+  hasBlob: hasVercelBlob || hasAwsS3Storage,
  hasAuth: (process.env.AUTH_SECRET ?? '').length > 0,
  hasAdminUser: (
    (process.env.ADMIN_EMAIL ?? '').length > 0 &&