Abstract blob service, add core S3 functionality

README.md

@@ -72,13 +72,14 @@ Installation
 
 #### AWS S3
 
-1. [Create bucket](https://s3.console.aws.amazon.com/s3) with "Block all public access" turned off
+1. [Create bucket](https://s3.console.aws.amazon.com/s3) with "ACLs enabled," and "Block all public access" turned off
    - Setup CORS:
      ```
      [{
-      "AllowedHeaders": [],
+      "AllowedHeaders": ["*"],
         "AllowedMethods": [
-          "GET"
+         "GET",
+         "PUT"
         ],
         "AllowedOrigins": [
           "http://localhost:*",
@@ -92,10 +93,10 @@ Installation
      - `NEXT_PUBLIC_S3_BUCKET`
      - `NEXT_PUBLIC_S3_REGION`
 2. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for client uploads (JSON editor recommended)
-   - Action: `s3:PutObject`
+   - Action: `s3:PutObject`, `s3:PutObjectACL`
-   - Resource: `arn:aws:s3:::{BUCKET_NAME}/uploads/*`
+   - Resource: `arn:aws:s3:::{BUCKET_NAME}/upload-*`
 3. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for admin actions (JSON editor recommended)
-   - Action: `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject`
+   - Action: `s3:PutObject`, `s3:PutObjectACL`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject`
    - Resource: `arn:aws:s3:::{BUCKET_NAME}`, `arn:aws:s3:::{BUCKET_NAME}/*`
 4. [Create IAM user](https://console.aws.amazon.com/iam/home#/users) for upload policy (by choosing "Attach policies directly"), create access key under "Security credentials," choose "Application running outside AWS," and store credentials
    - `NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY`

package.json

@@ -9,6 +9,7 @@
     "analyze": "ANALYZE=true next build"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.456.0",
     "@next/bundle-analyzer": "14.0.3",
     "@tailwindcss/forms": "^0.5.7",
     "@testing-library/jest-dom": "^6.1.4",

src/cache/index.ts

@@ -20,7 +20,7 @@ import {
   getUniqueFilmSimulations,
   getPhotosFilmSimulationDateRange,
   getPhotosFilmSimulationCount,
-} from '@/services/postgres';
+} from '@/services/vercel-postgres';
 import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo';
 import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob';
 import type { Session } from 'next-auth';

src/photo/PhotoUpload.tsx

@@ -56,7 +56,7 @@ export default function PhotoUpload({
                   blob,
                   extension,
                 )
-                  .then(({ url }) => {
+                  .then(url => {
                     if (isLastBlob) {
                       // Refresh page to update upload list,
                       // relevant to upload count in nav

src/photo/actions.ts

@@ -7,7 +7,7 @@ import {
   sqlUpdatePhoto,
   sqlRenamePhotoTagGlobally,
   getPhoto,
-} from '@/services/postgres';
+} from '@/services/vercel-postgres';
 import {
   PhotoFormData,
   convertFormDataToPhotoDbInsert,
@@ -16,7 +16,7 @@ import {
 import { redirect } from 'next/navigation';
 import {
   convertUploadToPhoto,
-  deleteBlobPhoto,
+  deleteBlobUrl,
 } from '@/services/blob';
 import {
   revalidateAdminPaths,
@@ -52,7 +52,7 @@ export async function updatePhotoAction(formData: FormData) {
 
 export async function deletePhotoAction(formData: FormData) {
   await Promise.all([
-    deleteBlobPhoto(formData.get('url') as string),
+    deleteBlobUrl(formData.get('url') as string),
     sqlDeletePhoto(formData.get('id') as string),
   ]);
 
@@ -80,7 +80,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) {
 }
 
 export async function deleteBlobPhotoAction(formData: FormData) {
-  await deleteBlobPhoto(formData.get('url') as string);
+  await deleteBlobUrl(formData.get('url') as string);
 
   revalidateAdminPaths();
 

src/photo/server.ts

@@ -1,4 +1,7 @@
-import { getExtensionFromBlobUrl, getIdFromBlobUrl } from '@/services/blob';
+import {
+  getExtensionFromBlobUrl,
+  getIdFromBlobUrl,
+} from '@/services/blob';
 import { convertExifToFormData } from '@/photo/form';
 import {
   getFujifilmSimulationFromMakerNote,

src/services/blob.ts

@@ -1,82 +0,0 @@
-import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
-import { copy, del, list } from '@vercel/blob';
-import { upload } from '@vercel/blob/client';
-
-const STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
-  /^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
-)?.[1].toLowerCase();
-
-export const BLOB_BASE_URL =
-  `https://${STORE_ID}.public.blob.vercel-storage.com`;
-
-const PREFIX_UPLOAD = 'upload';
-const PREFIX_PHOTO = 'photo';
-
-const REGEX_UPLOAD_PATH = new RegExp(
-  `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
-  'i',
-);
-
-const REGEX_UPLOAD_ID = new RegExp(
-  `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
-  'i',
-);
-
-export const pathForBlobUrl = (url: string) =>
-  url.replace(`${BLOB_BASE_URL}/`, '');
-
-export const getExtensionFromBlobUrl = (url: string) =>
-  url.match(/.([a-z]{1,4})$/i)?.[1];
-
-export const getIdFromBlobUrl = (url: string) =>
-  url.match(REGEX_UPLOAD_ID)?.[1];
-
-export const isUploadPathnameValid = (pathname?: string) =>
-  pathname?.match(REGEX_UPLOAD_PATH);
-
-export const uploadPhotoFromClient = async (
-  file: File | Blob,
-  extension = 'jpg',
-) =>
-  upload(
-    `${PREFIX_UPLOAD}.${extension}`,
-    file,
-    {
-      access: 'public',
-      handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
-    },
-  );
-
-export const convertUploadToPhoto = async (
-  uploadUrl: string,
-  photoId?: string,
-) => {
-  const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
-  const fileExtension = getExtensionFromBlobUrl(uploadUrl) ?? 'jpg';
-  const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
-
-  const { url } = await copy(
-    uploadUrl,
-    photoUrl,
-    {
-      access: 'public',
-      ...photoId && { addRandomSuffix: false },
-    }
-  );
-
-  if (url) {
-    await del(uploadUrl);
-  }
-
-  return url;
-};
-
-export const deleteBlobPhoto = (url: string) => del(url);
-
-export const getBlobUploadUrls = () =>
-  list({ prefix: `${PREFIX_UPLOAD}-` })
-    .then(({ blobs }) => blobs.map(({ url }) => url));
-
-export const getBlobPhotoUrls = () =>
-  list({ prefix: `${PREFIX_PHOTO}-` })
-    .then(({ blobs }) => blobs.map(({ url }) => url));

src/services/blob/aws-s3.ts

@@ -0,0 +1,93 @@
+import { generateNanoid } from '@/utility/nanoid';
+import {
+  S3Client,
+  CopyObjectCommand,
+  DeleteObjectCommand,
+  ListObjectsCommand,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3';
+
+const S3_BUCKET = process.env.NEXT_PUBLIC_S3_BUCKET ?? '';
+const S3_REGION = process.env.NEXT_PUBLIC_S3_REGION ?? '';
+const S3_UPLOAD_ACCESS_KEY =
+  process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '';
+const S3_UPLOAD_SECRET_ACCESS_KEY =
+  process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '';
+const S3_ADMIN_ACCESS_KEY =
+  process.env.S3_ADMIN_ACCESS_KEY;
+const S3_ADMIN_SECRET_ACCESS_KEY =
+  process.env.S3_ADMIN_SECRET_ACCESS_KEY;
+
+export const HAS_AWS_S3_STORAGE =
+  S3_BUCKET.length > 0 &&
+  S3_REGION.length > 0 &&
+  S3_UPLOAD_ACCESS_KEY.length > 0 &&
+  S3_UPLOAD_SECRET_ACCESS_KEY.length > 0;
+
+const client = new S3Client({
+  region: S3_REGION,
+  credentials: {
+    // Fallback on upload credentials if admin credentials are not available
+    accessKeyId: S3_ADMIN_ACCESS_KEY ?? S3_UPLOAD_ACCESS_KEY,
+    secretAccessKey: S3_ADMIN_SECRET_ACCESS_KEY ?? S3_UPLOAD_SECRET_ACCESS_KEY,
+  },
+});
+
+export const AWS_S3_BASE_URL =
+  `https://${S3_BUCKET}.s3.${S3_REGION}.amazonaws.com`;
+
+export const isUrlFromAwsS3 = (url: string) =>
+  url.startsWith(AWS_S3_BASE_URL);
+
+const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`;
+
+export const awsS3UploadFromClient = async (
+  file: File | Blob,
+  fileName: string,
+  extension: string,
+  addRandomSuffix?: boolean,
+) => {
+  const Key = addRandomSuffix
+    ? `${fileName}-${generateNanoid()}.${extension}`
+    : `${fileName}.${extension}`;
+  return client.send(new PutObjectCommand({
+    Bucket: S3_BUCKET,
+    Key,
+    Body: file,
+    ACL: 'public-read',
+  }))
+    .then(() => urlForKey(Key));
+};
+
+export const awsS3Copy = async (
+  fileNameSource: string,
+  fileNameDestination: string,
+  addRandomSuffix?: boolean,
+) => {
+  const name = fileNameSource.split('.')[0];
+  const extension = fileNameSource.split('.')[1];
+  const Key = addRandomSuffix
+    ? `${name}-${generateNanoid()}.${extension}`
+    : fileNameDestination;
+  return client.send(new CopyObjectCommand({
+    Bucket: S3_BUCKET,
+    CopySource: fileNameSource,
+    Key,
+    ACL: 'public-read',
+  }))
+    .then(() => urlForKey(fileNameDestination));
+};
+
+export const awsS3Delete = async (Key: string) => {
+  client.send(new DeleteObjectCommand({
+    Bucket: S3_BUCKET,
+    Key,
+  }));
+};
+
+export const awsS3List = async (Prefix: string) =>
+  client.send(new ListObjectsCommand({
+    Bucket: S3_BUCKET,
+    Prefix,
+  }))
+    .then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []);

src/services/blob/index.ts

@@ -0,0 +1,86 @@
+import {
+  VERCEL_BLOB_BASE_URL,
+  vercelBlobCopy,
+  vercelBlobDelete,
+  vercelBlobList,
+  vercelBlobUploadFromClient,
+} from './vercel-blob';
+import {
+  AWS_S3_BASE_URL,
+  HAS_AWS_S3_STORAGE,
+  awsS3Copy,
+  awsS3Delete,
+  awsS3List,
+  awsS3UploadFromClient,
+  isUrlFromAwsS3,
+} from './aws-s3';
+
+const PREFIX_UPLOAD = 'upload';
+const PREFIX_PHOTO = 'photo';
+const BLOB_BASE_URL = AWS_S3_BASE_URL ?? VERCEL_BLOB_BASE_URL;
+
+const REGEX_UPLOAD_PATH = new RegExp(
+  `(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
+  'i',
+);
+
+const REGEX_UPLOAD_ID = new RegExp(
+  `.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
+  'i',
+);
+
+export const pathForBlobUrl = (url: string) =>
+  url.replace(`${BLOB_BASE_URL}/`, '');
+
+export const getExtensionFromBlobUrl = (url: string) =>
+  url.match(/.([a-z]{1,4})$/i)?.[1];
+
+export const getIdFromBlobUrl = (url: string) =>
+  url.match(REGEX_UPLOAD_ID)?.[1];
+
+export const isUploadPathnameValid = (pathname?: string) =>
+  pathname?.match(REGEX_UPLOAD_PATH);
+
+const getFileNameFromBlobUrl = (url: string) =>
+  (new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? '';
+
+export const uploadPhotoFromClient = async (
+  file: File | Blob,
+  extension = 'jpg',
+) => HAS_AWS_S3_STORAGE
+  ? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true)
+  : vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`);
+
+export const convertUploadToPhoto = async (
+  uploadUrl: string,
+  photoId?: string,
+): Promise<string> => {
+  const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
+  const fileExtension = getExtensionFromBlobUrl(uploadUrl);
+  const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
+
+  const url = await (HAS_AWS_S3_STORAGE
+    ? awsS3Copy(uploadUrl, photoUrl, photoId === undefined)
+    : vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined));
+
+  if (url) {
+    await (HAS_AWS_S3_STORAGE
+      ? awsS3Delete(getFileNameFromBlobUrl(uploadUrl))
+      : vercelBlobDelete(uploadUrl));
+  }
+
+  return url;
+};
+
+export const deleteBlobUrl = (url: string) =>
+  HAS_AWS_S3_STORAGE && isUrlFromAwsS3(url)
+    ? awsS3Delete(getFileNameFromBlobUrl(url))
+    : vercelBlobDelete(url);
+
+export const getBlobUploadUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
+  ? awsS3List(`${PREFIX_UPLOAD}-`)
+  : vercelBlobList(`${PREFIX_UPLOAD}-`);
+
+export const getBlobPhotoUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
+  ? awsS3List(`${PREFIX_PHOTO}-`)
+  : vercelBlobList(`${PREFIX_PHOTO}-`);

src/services/blob/vercel-blob.ts

@@ -0,0 +1,44 @@
+import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
+import { copy, del, list } from '@vercel/blob';
+import { upload } from '@vercel/blob/client';
+
+const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
+  /^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
+)?.[1].toLowerCase();
+
+export const VERCEL_BLOB_BASE_URL =
+  `https://${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`;
+
+export const vercelBlobUploadFromClient = async (
+  file: File | Blob,
+  fileName: string,
+) =>
+  upload(
+    fileName,
+    file,
+    {
+      access: 'public',
+      handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
+    },
+  )
+    .then(({ url }) => url);
+
+export const vercelBlobCopy = (
+  fileNameSource: string,
+  fileNameDestination: string,
+  addRandomSuffix?: boolean,
+): Promise<string> =>
+  copy(
+    fileNameSource,
+    fileNameDestination,
+    {
+      access: 'public',
+      addRandomSuffix,
+    },
+  )
+    .then(({ url }) => url);
+
+export const vercelBlobDelete = (fileName: string) => del(fileName);
+
+export const vercelBlobList = (prefix: string) => list({ prefix })
+  .then(({ blobs }) => blobs.map(({ url }) => url));

src/site/config.ts

@@ -36,10 +36,10 @@ const hasVercelBlob = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0;
 const hasAwsS3Storage =
   (process.env.NEXT_PUBLIC_S3_BUCKET ?? '').length > 0 &&
   (process.env.NEXT_PUBLIC_S3_REGION ?? '').length > 0 &&
-  (process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_ID ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '').length > 0 &&
-  (process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET ?? '').length > 0 &&
+  (process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '').length > 0 &&
-  (process.env.S3_ADMIN_ACCESS_ID ?? '').length > 0 &&
+  (process.env.S3_ADMIN_ACCESS_KEY ?? '').length > 0 &&
-  (process.env.S3_ADMIN_ACCESS_SECRET ?? '').length > 0;
+  (process.env.S3_ADMIN_SECRET_ACCESS_KEY ?? '').length > 0;
   
 
 // SETTINGS