From f7de56a7b9ef748926b6691325fd0417667ae44c Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 27 Aug 2025 20:46:57 -0500 Subject: [PATCH] Refine storage config --- README.md | 35 +++++++++++++------------- next.config.ts | 37 +++++++++++++++------------- src/app/config.ts | 4 ++- src/platforms/storage/index.ts | 8 +++--- src/platforms/storage/minio.ts | 19 +++++++------- src/platforms/storage/vercel-blob.ts | 4 +-- 6 files changed, 57 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 4361e9d5..d5d479cc 100644 --- a/README.md +++ b/README.md @@ -278,18 +278,18 @@ Only one storage adapter—Vercel Blob, Cloudflare R2, AWS S3, or MinIO—can be ### MinIO -MinIO is an S3-compatible object storage server that you can host yourself, giving you complete control over your photo storage. +MinIO is a self-hosted S3-compatible object storage server. -### 1. Server and Public Bucket Setup +### 1. Server/bucket setup -First, install and deploy the MinIO server, and then create a bucket with public read access. +First, install and deploy the MinIO server, then create a bucket with public read access. -- **Install MinIO:** Follow the official documentation to [install and deploy MinIO](https://min.io/docs/minio/linux/operations/install-deploy-manage/deploy-minio-single-node-single-drive.html) on your server. -- **Create a bucket:** +- **Install MinIO:** [Follow official documentation](https://min.io/docs/minio/linux/operations/install-deploy-manage/deploy-minio-single-node-single-drive.html) to install and deploy MinIO. +- **Create bucket:** ```bash mc mb myminio/{BUCKET_NAME} ``` -- **Set public read policy:** Create a file named `bucket-policy.json` with the following content to allow read-only access to everyone: +- **Set public read policy:** Create file named `bucket-policy.json` with the following content to allow read-only access: ```json { "Version": "2012-10-17", @@ -311,22 +311,22 @@ First, install and deploy the MinIO server, and then create a bucket with public ] } ``` - Now, apply this policy to your bucket: + Next, apply this policy to your bucket: ```bash mc policy set myminio/photos bucket-policy.json ``` - **Store public configuration:** Set the following public environment variables for your application: - - `NEXT_PUBLIC_MINIO_BUCKET`: Your bucketname - - `NEXT_PUBLIC_MINIO_ENDPOINT`: MinIO server endpoint (e.g., "minio.yourdomain.com") + - `NEXT_PUBLIC_MINIO_BUCKET`: Bucket name + - `NEXT_PUBLIC_MINIO_DOMAIN`: MinIO server endpoint, e.g., "minio.yourdomain.com" - `NEXT_PUBLIC_MINIO_PORT`: (optional) - `NEXT_PUBLIC_MINIO_DISABLE_SSL`: Set to `1` to disable SSL (defaults to HTTPS) -### 2. Create a User with Restricted Permissions +### 2. Create user with restricted permissions -Next, create a dedicated user and a policy that grants permissions to manage objects within the `{BUCKET_NAME}` bucket. +Create a dedicated user and a policy that grants permission to manage objects within your `BUCKET_NAME`. -- **Define the user policy:** Create a file named `user-policy.json`. This policy will allow the user to list the bucket contents and to get, put, and delete objects within it. +- **Define user policy:** Create file named `user-policy.json`. This policy will allow the user to list the bucket contents and to get, put, and delete objects within it. ```json { "Version": "2012-10-17", @@ -347,20 +347,21 @@ Next, create a dedicated user and a policy that grants permissions to manage obj ] } ``` -- **Create the policy:** Add the policy to MinIO and give it a name, for example, `photos-manager-policy`. +- **Create policy:** Add named policy to MinIO. ```bash mc admin policy add myminio photos-manager-policy user-policy.json ``` -- **Create a new user:** Create a new user with an access key and a secret key. +- **Create user:** Create new user with access key and secret key. ```bash mc admin user add myminio {MINIO_ACCESS_KEY} {MINIO_SECRET_ACCESS_KEY} - ```- **Attach the policy to the user:** Assign the `photos-manager-policy` to the new user. + ``` +- **Attach policy to user:** Assign `photos-manager-policy` to the user. ```bash mc admin policy set myminio photos-manager-policy user=MINIO_ACCESS_KEY - ```- **Store private credentials:** Set the following private environment variables for your application. ⚠️ **Ensure these access keys are not prefixed with `NEXT_PUBLIC`**. + ``` +- **Store private credentials:** Set the following private environment variables for your application. ⚠️ **Ensure these access keys are not prefixed with `NEXT_PUBLIC`**. - `MINIO_ACCESS_KEY`: Your MINIO_ACCESS_KEY - `MINIO_SECRET_ACCESS_KEY`: Your MINIO_SECRET_ACCESS_KEY - ## Alternate database providers (experimental) diff --git a/next.config.ts b/next.config.ts index 69f1e63c..8ea89398 100644 --- a/next.config.ts +++ b/next.config.ts @@ -22,22 +22,22 @@ const HOSTNAME_AWS_S3 = : undefined; const HOSTNAME_MINIO = - process.env.NEXT_PUBLIC_MINIO_ENDPOINT - ? process.env.NEXT_PUBLIC_MINIO_ENDPOINT - : undefined; + process.env.NEXT_PUBLIC_MINIO_DOMAIN; +const MINIO_PORT = + process.env.NEXT_PUBLIC_MINIO_PORT; +const MINIO_USE_SSL = + process.env.NEXT_PUBLIC_MINIO_DISABLE_SSL !== '1'; -const generateRemotePattern = (_hostname: string, useSSL = true) => { - const hostname = removeUrlProtocol(_hostname)!; - - const [hostnamePart, portPart] = hostname.split(':'); - - return { - protocol: useSSL ? 'https' : 'http', - hostname: hostnamePart, - port: portPart || '', - pathname: '/**', - } as const; -}; +const generateRemotePattern = ( + hostname: string, + port?: string, + useSSL = true, +): RemotePattern => ({ + protocol: useSSL ? 'https' : 'http', + hostname: removeUrlProtocol(hostname)!, + port, + pathname: '/**', +}); const remotePatterns: RemotePattern[] = []; @@ -51,8 +51,11 @@ if (HOSTNAME_AWS_S3) { remotePatterns.push(generateRemotePattern(HOSTNAME_AWS_S3)); } if (HOSTNAME_MINIO) { - const useSSL = process.env.NEXT_PUBLIC_MINIO_DISABLE_SSL !== '1'; - remotePatterns.push(generateRemotePattern(HOSTNAME_MINIO, useSSL)); + remotePatterns.push(generateRemotePattern( + HOSTNAME_MINIO, + MINIO_PORT, + MINIO_USE_SSL, + )); } const LOCALE = process.env.NEXT_PUBLIC_LOCALE || 'en-us'; diff --git a/src/app/config.ts b/src/app/config.ts index e99bb578..f6d626e9 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -186,9 +186,11 @@ export const HAS_AWS_S3_STORAGE = Boolean(process.env.AWS_S3_ACCESS_KEY) && Boolean(process.env.AWS_S3_SECRET_ACCESS_KEY); +// STORAGE: MINIO +// Includes separate check for client-side usage, i.e., url construction export const HAS_MINIO_STORAGE_CLIENT = Boolean(process.env.NEXT_PUBLIC_MINIO_BUCKET) && - Boolean(process.env.NEXT_PUBLIC_MINIO_ENDPOINT); + Boolean(process.env.NEXT_PUBLIC_MINIO_DOMAIN); export const HAS_MINIO_STORAGE = HAS_MINIO_STORAGE_CLIENT && Boolean(process.env.MINIO_ACCESS_KEY) && diff --git a/src/platforms/storage/index.ts b/src/platforms/storage/index.ts index 570bc6bd..b6358c7e 100644 --- a/src/platforms/storage/index.ts +++ b/src/platforms/storage/index.ts @@ -103,7 +103,7 @@ const REGEX_UPLOAD_ID = new RegExp( 'i', ); -export const fileNameForStorageUrl = (url: string) => { +export const getFilePathFromStorageUrl = (url: string) => { switch (storageTypeFromUrl(url)) { case 'vercel-blob': return url.replace(`${VERCEL_BLOB_BASE_URL}/`, ''); @@ -209,11 +209,11 @@ export const deleteFile = (url: string) => { case 'vercel-blob': return vercelBlobDelete(url); case 'cloudflare-r2': - return cloudflareR2Delete(getFileNameFromStorageUrl(url)); + return cloudflareR2Delete(getFilePathFromStorageUrl(url)); case 'aws-s3': - return awsS3Delete(getFileNameFromStorageUrl(url)); + return awsS3Delete(getFilePathFromStorageUrl(url)); case 'minio': - return minioDelete(fileNameForStorageUrl(url)); + return minioDelete(getFilePathFromStorageUrl(url)); } }; diff --git a/src/platforms/storage/minio.ts b/src/platforms/storage/minio.ts index 2e85d7e2..b0feec7f 100644 --- a/src/platforms/storage/minio.ts +++ b/src/platforms/storage/minio.ts @@ -2,30 +2,31 @@ import { S3Client, CopyObjectCommand, ListObjectsCommand, - PutObjectCommand, DeleteObjectsCommand, + PutObjectCommand, DeleteObjectCommand, } from '@aws-sdk/client-s3'; import { StorageListResponse, generateStorageId } from '.'; import { formatBytesToMB } from '@/utility/number'; const MINIO_BUCKET = process.env.NEXT_PUBLIC_MINIO_BUCKET ?? ''; -const MINIO_ENDPOINT = process.env.NEXT_PUBLIC_MINIO_ENDPOINT ?? ''; +const MINIO_DOMAIN = process.env.NEXT_PUBLIC_MINIO_DOMAIN ?? ''; const MINIO_PORT = process.env.NEXT_PUBLIC_MINIO_PORT ?? ''; const MINIO_DISABLE_SSL = process.env.NEXT_PUBLIC_MINIO_DISABLE_SSL === '1'; const MINIO_ACCESS_KEY = process.env.MINIO_ACCESS_KEY ?? ''; const MINIO_SECRET_ACCESS_KEY = process.env.MINIO_SECRET_ACCESS_KEY ?? ''; -export const MINIO_BASE_URL = MINIO_BUCKET && MINIO_ENDPOINT - ? `${MINIO_DISABLE_SSL ? 'http' : 'https'}://${MINIO_ENDPOINT}${ - MINIO_PORT ? `:${MINIO_PORT}` : '' - }/${MINIO_BUCKET}` +const PROTOCOL = MINIO_DISABLE_SSL ? 'http' : 'https'; +const ENDPOINT = MINIO_BUCKET && MINIO_DOMAIN + ? `${PROTOCOL}://${MINIO_DOMAIN}${MINIO_PORT ? `:${MINIO_PORT}` : ''}` + : undefined; + +export const MINIO_BASE_URL = ENDPOINT + ? `${ENDPOINT}/${MINIO_BUCKET}` : undefined; export const minioClient = () => new S3Client({ region: 'us-east-1', - endpoint: `${MINIO_DISABLE_SSL ? 'http' : 'https'}://${MINIO_ENDPOINT}${ - MINIO_PORT ? `:${MINIO_PORT}` : '' - }`, + endpoint: ENDPOINT, credentials: { accessKeyId: MINIO_ACCESS_KEY, secretAccessKey: MINIO_SECRET_ACCESS_KEY, diff --git a/src/platforms/storage/vercel-blob.ts b/src/platforms/storage/vercel-blob.ts index 698f856e..1633ad6c 100644 --- a/src/platforms/storage/vercel-blob.ts +++ b/src/platforms/storage/vercel-blob.ts @@ -1,7 +1,7 @@ import { PATH_API_VERCEL_BLOB_UPLOAD } from '@/app/path'; import { copy, del, list, put } from '@vercel/blob'; import { upload } from '@vercel/blob/client'; -import { fileNameForStorageUrl, StorageListResponse } from '.'; +import { getFilePathFromStorageUrl, StorageListResponse } from '.'; import { formatBytesToMB } from '@/utility/number'; const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match( @@ -55,7 +55,7 @@ export const vercelBlobList = ( ): Promise => list({ prefix }) .then(({ blobs }) => blobs.map(({ url, uploadedAt, size }) => ({ url, - fileName: fileNameForStorageUrl(url), + fileName: getFilePathFromStorageUrl(url), uploadedAt, size: formatBytesToMB(size), })));