strtus/Vercel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Abstract blob service, add core S3 functionality

Browse Source
This commit is contained in:
Sam Becker 2023-11-26 18:25:24 -06:00
parent 25941329db
commit fe992c0e17
13 changed files with 1292 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -72,13 +72,14 @@ Installation
#### AWS S3
1. [Create bucket](https://s3.console.aws.amazon.com/s3) with "Block all public access" turned off
1. [Create bucket](https://s3.console.aws.amazon.com/s3) with "ACLs enabled," and "Block all public access" turned off
- Setup CORS:
```
[{
"AllowedHeaders": [],
"AllowedHeaders": ["*"],
"AllowedMethods": [
"GET"
"GET",
"PUT"
],
"AllowedOrigins": [
"http://localhost:*",
@ -92,10 +93,10 @@ Installation
- `NEXT_PUBLIC_S3_BUCKET`
- `NEXT_PUBLIC_S3_REGION`
2. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for client uploads (JSON editor recommended)
- Action: `s3:PutObject`
- Resource: `arn:aws:s3:::{BUCKET_NAME}/uploads/*`
- Action: `s3:PutObject`, `s3:PutObjectACL`
- Resource: `arn:aws:s3:::{BUCKET_NAME}/upload-*`
3. [Create IAM policy](https://console.aws.amazon.com/iam/home#/policies) for admin actions (JSON editor recommended)
- Action: `s3:PutObject`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject`
- Action: `s3:PutObject`, `s3:PutObjectACL`, `s3:GetObject`, `s3:ListBucket`, `s3:DeleteObject`
- Resource: `arn:aws:s3:::{BUCKET_NAME}`, `arn:aws:s3:::{BUCKET_NAME}/*`
4. [Create IAM user](https://console.aws.amazon.com/iam/home#/users) for upload policy (by choosing "Attach policies directly"), create access key under "Security credentials," choose "Application running outside AWS," and store credentials
- `NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY`

1
package.json
View File

@ -9,6 +9,7 @@
"analyze": "ANALYZE=true next build"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.456.0",
"@next/bundle-analyzer": "14.0.3",
"@tailwindcss/forms": "^0.5.7",
"@testing-library/jest-dom": "^6.1.4",

1047
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

2
src/cache/index.ts vendored
View File

@ -20,7 +20,7 @@ import {
getUniqueFilmSimulations,
getPhotosFilmSimulationDateRange,
getPhotosFilmSimulationCount,
} from '@/services/postgres';
} from '@/services/vercel-postgres';
import { parseCachedPhotoDates, parseCachedPhotosDates } from '@/photo';
import { getBlobPhotoUrls, getBlobUploadUrls } from '@/services/blob';
import type { Session } from 'next-auth';

2
src/photo/PhotoUpload.tsx
View File

@ -56,7 +56,7 @@ export default function PhotoUpload({
blob,
extension,
)
.then(({ url }) => {
.then(url => {
if (isLastBlob) {
// Refresh page to update upload list,
// relevant to upload count in nav

8
src/photo/actions.ts
View File

@ -7,7 +7,7 @@ import {
sqlUpdatePhoto,
sqlRenamePhotoTagGlobally,
getPhoto,
} from '@/services/postgres';
} from '@/services/vercel-postgres';
import {
PhotoFormData,
convertFormDataToPhotoDbInsert,
@ -16,7 +16,7 @@ import {
import { redirect } from 'next/navigation';
import {
convertUploadToPhoto,
deleteBlobPhoto,
deleteBlobUrl,
} from '@/services/blob';
import {
revalidateAdminPaths,
@ -52,7 +52,7 @@ export async function updatePhotoAction(formData: FormData) {
export async function deletePhotoAction(formData: FormData) {
await Promise.all([
deleteBlobPhoto(formData.get('url') as string),
deleteBlobUrl(formData.get('url') as string),
sqlDeletePhoto(formData.get('id') as string),
]);
@ -80,7 +80,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) {
}
export async function deleteBlobPhotoAction(formData: FormData) {
await deleteBlobPhoto(formData.get('url') as string);
await deleteBlobUrl(formData.get('url') as string);
revalidateAdminPaths();

5
src/photo/server.ts
View File

@ -1,4 +1,7 @@
import { getExtensionFromBlobUrl, getIdFromBlobUrl } from '@/services/blob';
import {
getExtensionFromBlobUrl,
getIdFromBlobUrl,
} from '@/services/blob';
import { convertExifToFormData } from '@/photo/form';
import {
getFujifilmSimulationFromMakerNote,

82
src/services/blob.ts
View File

@ -1,82 +0,0 @@
import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
import { copy, del, list } from '@vercel/blob';
import { upload } from '@vercel/blob/client';
const STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
/^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
)?.[1].toLowerCase();
export const BLOB_BASE_URL =
`https://${STORE_ID}.public.blob.vercel-storage.com`;
const PREFIX_UPLOAD = 'upload';
const PREFIX_PHOTO = 'photo';
const REGEX_UPLOAD_PATH = new RegExp(
`(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
'i',
);
const REGEX_UPLOAD_ID = new RegExp(
`.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
'i',
);
export const pathForBlobUrl = (url: string) =>
url.replace(`${BLOB_BASE_URL}/`, '');
export const getExtensionFromBlobUrl = (url: string) =>
url.match(/.([a-z]{1,4})$/i)?.[1];
export const getIdFromBlobUrl = (url: string) =>
url.match(REGEX_UPLOAD_ID)?.[1];
export const isUploadPathnameValid = (pathname?: string) =>
pathname?.match(REGEX_UPLOAD_PATH);
export const uploadPhotoFromClient = async (
file: File | Blob,
extension = 'jpg',
) =>
upload(
`${PREFIX_UPLOAD}.${extension}`,
file,
{
access: 'public',
handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
},
);
export const convertUploadToPhoto = async (
uploadUrl: string,
photoId?: string,
) => {
const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
const fileExtension = getExtensionFromBlobUrl(uploadUrl) ?? 'jpg';
const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
const { url } = await copy(
uploadUrl,
photoUrl,
{
access: 'public',
...photoId && { addRandomSuffix: false },
}
);
if (url) {
await del(uploadUrl);
}
return url;
};
export const deleteBlobPhoto = (url: string) => del(url);
export const getBlobUploadUrls = () =>
list({ prefix: `${PREFIX_UPLOAD}-` })
.then(({ blobs }) => blobs.map(({ url }) => url));
export const getBlobPhotoUrls = () =>
list({ prefix: `${PREFIX_PHOTO}-` })
.then(({ blobs }) => blobs.map(({ url }) => url));

93
src/services/blob/aws-s3.ts Normal file
View File

@ -0,0 +1,93 @@
import { generateNanoid } from '@/utility/nanoid';
import {
S3Client,
CopyObjectCommand,
DeleteObjectCommand,
ListObjectsCommand,
PutObjectCommand,
} from '@aws-sdk/client-s3';
const S3_BUCKET = process.env.NEXT_PUBLIC_S3_BUCKET ?? '';
const S3_REGION = process.env.NEXT_PUBLIC_S3_REGION ?? '';
const S3_UPLOAD_ACCESS_KEY =
process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '';
const S3_UPLOAD_SECRET_ACCESS_KEY =
process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '';
const S3_ADMIN_ACCESS_KEY =
process.env.S3_ADMIN_ACCESS_KEY;
const S3_ADMIN_SECRET_ACCESS_KEY =
process.env.S3_ADMIN_SECRET_ACCESS_KEY;
export const HAS_AWS_S3_STORAGE =
S3_BUCKET.length > 0 &&
S3_REGION.length > 0 &&
S3_UPLOAD_ACCESS_KEY.length > 0 &&
S3_UPLOAD_SECRET_ACCESS_KEY.length > 0;
const client = new S3Client({
region: S3_REGION,
credentials: {
// Fallback on upload credentials if admin credentials are not available
accessKeyId: S3_ADMIN_ACCESS_KEY ?? S3_UPLOAD_ACCESS_KEY,
secretAccessKey: S3_ADMIN_SECRET_ACCESS_KEY ?? S3_UPLOAD_SECRET_ACCESS_KEY,
},
});
export const AWS_S3_BASE_URL =
`https://${S3_BUCKET}.s3.${S3_REGION}.amazonaws.com`;
export const isUrlFromAwsS3 = (url: string) =>
url.startsWith(AWS_S3_BASE_URL);
const urlForKey = (key?: string) => `${AWS_S3_BASE_URL}/${key}`;
export const awsS3UploadFromClient = async (
file: File | Blob,
fileName: string,
extension: string,
addRandomSuffix?: boolean,
) => {
const Key = addRandomSuffix
? `${fileName}-${generateNanoid()}.${extension}`
: `${fileName}.${extension}`;
return client.send(new PutObjectCommand({
Bucket: S3_BUCKET,
Key,
Body: file,
ACL: 'public-read',
}))
.then(() => urlForKey(Key));
};
export const awsS3Copy = async (
fileNameSource: string,
fileNameDestination: string,
addRandomSuffix?: boolean,
) => {
const name = fileNameSource.split('.')[0];
const extension = fileNameSource.split('.')[1];
const Key = addRandomSuffix
? `${name}-${generateNanoid()}.${extension}`
: fileNameDestination;
return client.send(new CopyObjectCommand({
Bucket: S3_BUCKET,
CopySource: fileNameSource,
Key,
ACL: 'public-read',
}))
.then(() => urlForKey(fileNameDestination));
};
export const awsS3Delete = async (Key: string) => {
client.send(new DeleteObjectCommand({
Bucket: S3_BUCKET,
Key,
}));
};
export const awsS3List = async (Prefix: string) =>
client.send(new ListObjectsCommand({
Bucket: S3_BUCKET,
Prefix,
}))
.then((data) => data.Contents?.map(({ Key }) => urlForKey(Key)) ?? []);

86
src/services/blob/index.ts Normal file
View File

@ -0,0 +1,86 @@
import {
VERCEL_BLOB_BASE_URL,
vercelBlobCopy,
vercelBlobDelete,
vercelBlobList,
vercelBlobUploadFromClient,
} from './vercel-blob';
import {
AWS_S3_BASE_URL,
HAS_AWS_S3_STORAGE,
awsS3Copy,
awsS3Delete,
awsS3List,
awsS3UploadFromClient,
isUrlFromAwsS3,
} from './aws-s3';
const PREFIX_UPLOAD = 'upload';
const PREFIX_PHOTO = 'photo';
const BLOB_BASE_URL = AWS_S3_BASE_URL ?? VERCEL_BLOB_BASE_URL;
const REGEX_UPLOAD_PATH = new RegExp(
`(?:${PREFIX_UPLOAD})\.[a-z]{1,4}`,
'i',
);
const REGEX_UPLOAD_ID = new RegExp(
`.${PREFIX_UPLOAD}-([a-z0-9]+)\.[a-z]{1,4}$`,
'i',
);
export const pathForBlobUrl = (url: string) =>
url.replace(`${BLOB_BASE_URL}/`, '');
export const getExtensionFromBlobUrl = (url: string) =>
url.match(/.([a-z]{1,4})$/i)?.[1];
export const getIdFromBlobUrl = (url: string) =>
url.match(REGEX_UPLOAD_ID)?.[1];
export const isUploadPathnameValid = (pathname?: string) =>
pathname?.match(REGEX_UPLOAD_PATH);
const getFileNameFromBlobUrl = (url: string) =>
(new URL(url).pathname.match(/\/(.+)$/)?.[1]) ?? '';
export const uploadPhotoFromClient = async (
file: File | Blob,
extension = 'jpg',
) => HAS_AWS_S3_STORAGE
? awsS3UploadFromClient(file, PREFIX_UPLOAD, extension, true)
: vercelBlobUploadFromClient(file, `${PREFIX_UPLOAD}.${extension}`);
export const convertUploadToPhoto = async (
uploadUrl: string,
photoId?: string,
): Promise<string> => {
const fileName = photoId ? `${PREFIX_PHOTO}-${photoId}` : `${PREFIX_PHOTO}`;
const fileExtension = getExtensionFromBlobUrl(uploadUrl);
const photoUrl = `${fileName}.${fileExtension ?? 'jpg'}`;
const url = await (HAS_AWS_S3_STORAGE
? awsS3Copy(uploadUrl, photoUrl, photoId === undefined)
: vercelBlobCopy(uploadUrl, photoUrl, photoId === undefined));
if (url) {
await (HAS_AWS_S3_STORAGE
? awsS3Delete(getFileNameFromBlobUrl(uploadUrl))
: vercelBlobDelete(uploadUrl));
}
return url;
};
export const deleteBlobUrl = (url: string) =>
HAS_AWS_S3_STORAGE && isUrlFromAwsS3(url)
? awsS3Delete(getFileNameFromBlobUrl(url))
: vercelBlobDelete(url);
export const getBlobUploadUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
? awsS3List(`${PREFIX_UPLOAD}-`)
: vercelBlobList(`${PREFIX_UPLOAD}-`);
export const getBlobPhotoUrls = (): Promise<string[]> => HAS_AWS_S3_STORAGE
? awsS3List(`${PREFIX_PHOTO}-`)
: vercelBlobList(`${PREFIX_PHOTO}-`);

44
src/services/blob/vercel-blob.ts Normal file
View File

@ -0,0 +1,44 @@
import { PATH_ADMIN_UPLOAD_BLOB } from '@/site/paths';
import { copy, del, list } from '@vercel/blob';
import { upload } from '@vercel/blob/client';
const VERCEL_BLOB_STORE_ID = process.env.BLOB_READ_WRITE_TOKEN?.match(
/^vercel_blob_rw_([a-z0-9]+)_[a-z0-9]+$/i,
)?.[1].toLowerCase();
export const VERCEL_BLOB_BASE_URL =
`https://${VERCEL_BLOB_STORE_ID}.public.blob.vercel-storage.com`;
export const vercelBlobUploadFromClient = async (
file: File | Blob,
fileName: string,
) =>
upload(
fileName,
file,
{
access: 'public',
handleUploadUrl: PATH_ADMIN_UPLOAD_BLOB,
},
)
.then(({ url }) => url);
export const vercelBlobCopy = (
fileNameSource: string,
fileNameDestination: string,
addRandomSuffix?: boolean,
): Promise<string> =>
copy(
fileNameSource,
fileNameDestination,
{
access: 'public',
addRandomSuffix,
},
)
.then(({ url }) => url);
export const vercelBlobDelete = (fileName: string) => del(fileName);
export const vercelBlobList = (prefix: string) => list({ prefix })
.then(({ blobs }) => blobs.map(({ url }) => url));

0
src/services/postgres.ts → src/services/vercel-postgres.ts
View File

8
src/site/config.ts
View File

@ -36,10 +36,10 @@ const hasVercelBlob = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0;
const hasAwsS3Storage =
(process.env.NEXT_PUBLIC_S3_BUCKET ?? '').length > 0 &&
(process.env.NEXT_PUBLIC_S3_REGION ?? '').length > 0 &&
(process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_ID ?? '').length > 0 &&
(process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET ?? '').length > 0 &&
(process.env.S3_ADMIN_ACCESS_ID ?? '').length > 0 &&
(process.env.S3_ADMIN_ACCESS_SECRET ?? '').length > 0;
(process.env.NEXT_PUBLIC_S3_UPLOAD_ACCESS_KEY ?? '').length > 0 &&
(process.env.NEXT_PUBLIC_S3_UPLOAD_SECRET_ACCESS_KEY ?? '').length > 0 &&
(process.env.S3_ADMIN_ACCESS_KEY ?? '').length > 0 &&
(process.env.S3_ADMIN_SECRET_ACCESS_KEY ?? '').length > 0;
// SETTINGS