From de95b8c5f095ca3a5c347ccbd51fdb76135482b8 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 11:54:51 -0500 Subject: [PATCH 01/38] Add openai/ai foundations --- package.json | 2 + pnpm-lock.yaml | 489 ++++++++++++++++++++++++++- src/components/CanvasBlurCapture.tsx | 18 +- src/photo/form/PhotoForm.tsx | 6 + src/services/openai.ts | 37 ++ 5 files changed, 548 insertions(+), 4 deletions(-) create mode 100644 src/services/openai.ts diff --git a/package.json b/package.json index 4e81a298..035cf274 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@vercel/blob": "^0.22.1", "@vercel/postgres": "0.7.2", "@vercel/speed-insights": "^1.0.10", + "ai": "^3.0.13", "autoprefixer": "10.4.18", "camelcase-keys": "^9.1.3", "clsx": "^2.1.0", @@ -42,6 +43,7 @@ "next": "14.1.3", "next-auth": "5.0.0-beta.13", "next-themes": "^0.3.0", + "openai": "^4.29.2", "postcss": "8.4.35", "react": "18.2.0", "react-dom": "18.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f56bdd4..b0914cca 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,7 +58,10 @@ dependencies: version: 0.7.2 '@vercel/speed-insights': specifier: ^1.0.10 - version: 1.0.10(next@14.1.3)(react@18.2.0) + version: 1.0.10(next@14.1.3)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21) + ai: + specifier: ^3.0.13 + version: 3.0.13(react@18.2.0)(solid-js@1.8.15)(svelte@4.2.12)(vue@3.4.21)(zod@3.22.4) autoprefixer: specifier: 10.4.18 version: 10.4.18(postcss@8.4.35) @@ -104,6 +107,9 @@ dependencies: next-themes: specifier: ^0.3.0 version: 0.3.0(react-dom@18.2.0)(react@18.2.0) + openai: + specifier: ^4.29.2 + version: 4.29.2 postcss: specifier: 8.4.35 version: 8.4.35 @@ -2780,6 +2786,14 @@ packages: resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} dev: false + /@types/diff-match-patch@1.0.36: + resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==} + dev: false + + /@types/estree@1.0.5: + resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} + dev: false + /@types/graceful-fs@4.1.9: resolution: {integrity: sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==} dependencies: @@ -2825,6 +2839,19 @@ packages: resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} dev: false + /@types/node-fetch@2.6.11: + resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} + dependencies: + '@types/node': 20.11.26 + form-data: 4.0.0 + dev: false + + /@types/node@18.19.24: + resolution: {integrity: sha512-eghAz3gnbQbvnHqB+mgB2ZR3aH6RhdEmHGS48BnV75KceQPHqabkxKI0BbUSsqhqy2Ddhc2xD/VAR9ySZd57Lw==} + dependencies: + undici-types: 5.26.5 + dev: false + /@types/node@20.11.26: resolution: {integrity: sha512-YwOMmyhNnAWijOBQweOJnQPl068Oqd4K3OFbTc6AHJwzweUwwWG3GIFY74OKks2PJUDkQPeddOQES9mLn1CTEQ==} dependencies: @@ -3119,7 +3146,7 @@ packages: ws: 8.14.2(bufferutil@4.0.8)(utf-8-validate@6.0.3) dev: false - /@vercel/speed-insights@1.0.10(next@14.1.3)(react@18.2.0): + /@vercel/speed-insights@1.0.10(next@14.1.3)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21): resolution: {integrity: sha512-4uzdKB0RW6Ff2FkzshzjZ+RlJfLPxgm/00i0XXgxfMPhwnnsk92YgtqsxT9OcPLdJUyVU1DqFlSWWjIQMPkh0g==} requiresBuild: true peerDependencies: @@ -3145,6 +3172,81 @@ packages: dependencies: next: 14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 + svelte: 4.2.12 + vue: 3.4.21(typescript@5.4.2) + dev: false + + /@vue/compiler-core@3.4.21: + resolution: {integrity: sha512-MjXawxZf2SbZszLPYxaFCjxfibYrzr3eYbKxwpLR9EQN+oaziSu3qKVbwBERj1IFIB8OLUewxB5m/BFzi613og==} + dependencies: + '@babel/parser': 7.23.9 + '@vue/shared': 3.4.21 + entities: 4.5.0 + estree-walker: 2.0.2 + source-map-js: 1.0.2 + dev: false + + /@vue/compiler-dom@3.4.21: + resolution: {integrity: sha512-IZC6FKowtT1sl0CR5DpXSiEB5ayw75oT2bma1BEhV7RRR1+cfwLrxc2Z8Zq/RGFzJ8w5r9QtCOvTjQgdn0IKmA==} + dependencies: + '@vue/compiler-core': 3.4.21 + '@vue/shared': 3.4.21 + dev: false + + /@vue/compiler-sfc@3.4.21: + resolution: {integrity: sha512-me7epoTxYlY+2CUM7hy9PCDdpMPfIwrOvAXud2Upk10g4YLv9UBW7kL798TvMeDhPthkZ0CONNrK2GoeI1ODiQ==} + dependencies: + '@babel/parser': 7.23.9 + '@vue/compiler-core': 3.4.21 + '@vue/compiler-dom': 3.4.21 + '@vue/compiler-ssr': 3.4.21 + '@vue/shared': 3.4.21 + estree-walker: 2.0.2 + magic-string: 0.30.8 + postcss: 8.4.35 + source-map-js: 1.0.2 + dev: false + + /@vue/compiler-ssr@3.4.21: + resolution: {integrity: sha512-M5+9nI2lPpAsgXOGQobnIueVqc9sisBFexh5yMIMRAPYLa7+5wEJs8iqOZc1WAa9WQbx9GR2twgznU8LTIiZ4Q==} + dependencies: + '@vue/compiler-dom': 3.4.21 + '@vue/shared': 3.4.21 + dev: false + + /@vue/reactivity@3.4.21: + resolution: {integrity: sha512-UhenImdc0L0/4ahGCyEzc/pZNwVgcglGy9HVzJ1Bq2Mm9qXOpP8RyNTjookw/gOCUlXSEtuZ2fUg5nrHcoqJcw==} + dependencies: + '@vue/shared': 3.4.21 + dev: false + + /@vue/runtime-core@3.4.21: + resolution: {integrity: sha512-pQthsuYzE1XcGZznTKn73G0s14eCJcjaLvp3/DKeYWoFacD9glJoqlNBxt3W2c5S40t6CCcpPf+jG01N3ULyrA==} + dependencies: + '@vue/reactivity': 3.4.21 + '@vue/shared': 3.4.21 + dev: false + + /@vue/runtime-dom@3.4.21: + resolution: {integrity: sha512-gvf+C9cFpevsQxbkRBS1NpU8CqxKw0ebqMvLwcGQrNpx6gqRDodqKqA+A2VZZpQ9RpK2f9yfg8VbW/EpdFUOJw==} + dependencies: + '@vue/runtime-core': 3.4.21 + '@vue/shared': 3.4.21 + csstype: 3.1.3 + dev: false + + /@vue/server-renderer@3.4.21(vue@3.4.21): + resolution: {integrity: sha512-aV1gXyKSN6Rz+6kZ6kr5+Ll14YzmIbeuWe7ryJl5muJ4uwSwY/aStXTixx76TwkZFJLm1aAlA/HSWEJ4EyiMkg==} + peerDependencies: + vue: 3.4.21 + dependencies: + '@vue/compiler-ssr': 3.4.21 + '@vue/shared': 3.4.21 + vue: 3.4.21(typescript@5.4.2) + dev: false + + /@vue/shared@3.4.21: + resolution: {integrity: sha512-PuJe7vDIi6VYSinuEbUIQgMIRZGgM8e4R+G+/dQTk0X1NEdvgvvgv7m+rfmDH1gZzyA1OjjoWskvHlfRNfQf3g==} dev: false /abab@2.0.6: @@ -3152,6 +3254,13 @@ packages: deprecated: Use your platform's native atob() and btoa() methods instead dev: false + /abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + dependencies: + event-target-shim: 5.0.1 + dev: false + /acorn-globals@7.0.1: resolution: {integrity: sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q==} dependencies: @@ -3187,6 +3296,50 @@ packages: - supports-color dev: false + /agentkeepalive@4.5.0: + resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} + engines: {node: '>= 8.0.0'} + dependencies: + humanize-ms: 1.2.1 + dev: false + + /ai@3.0.13(react@18.2.0)(solid-js@1.8.15)(svelte@4.2.12)(vue@3.4.21)(zod@3.22.4): + resolution: {integrity: sha512-fDrYnVTdMJuS/qYUq0T/CX3WDuTfcZFie9LkgnoQ2layfUG2Wzh/mpfkfYXFEq/mqnpep3xUtECOB1weyyvwUg==} + engines: {node: '>=14.6'} + peerDependencies: + react: ^18.2.0 + solid-js: ^1.7.7 + svelte: ^3.0.0 || ^4.0.0 + vue: ^3.3.4 + zod: ^3.0.0 + peerDependenciesMeta: + react: + optional: true + solid-js: + optional: true + svelte: + optional: true + vue: + optional: true + zod: + optional: true + dependencies: + eventsource-parser: 1.0.0 + jsondiffpatch: 0.6.0 + nanoid: 3.3.6 + react: 18.2.0 + solid-js: 1.8.15 + solid-swr-store: 0.10.7(solid-js@1.8.15)(swr-store@0.10.6) + sswr: 2.0.0(svelte@4.2.12) + svelte: 4.2.12 + swr: 2.2.0(react@18.2.0) + swr-store: 0.10.6 + swrv: 1.0.4(vue@3.4.21) + vue: 3.4.21(typescript@5.4.2) + zod: 3.22.4 + zod-to-json-schema: 3.22.4(zod@3.22.4) + dev: false + /ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} dependencies: @@ -3424,6 +3577,12 @@ packages: dequal: 2.0.3 dev: false + /axobject-query@4.0.0: + resolution: {integrity: sha512-+60uv1hiVFhHZeO+Lz0RYzsVHy5Wr1ayX0mwda9KPDVLNJgZ1T9Ny7VmFbLDzxsH0D87I86vgj3gFrjTJUYznw==} + dependencies: + dequal: 2.0.3 + dev: false + /babel-jest@29.7.0(@babel/core@7.23.9): resolution: {integrity: sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3500,6 +3659,10 @@ packages: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} dev: false + /base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + dev: false + /binary-extensions@2.2.0: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} @@ -3655,11 +3818,20 @@ packages: supports-color: 7.2.0 dev: false + /chalk@5.3.0: + resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + dev: false + /char-regex@1.0.2: resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} engines: {node: '>=10'} dev: false + /charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + dev: false + /chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -3722,6 +3894,16 @@ packages: engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} dev: false + /code-red@1.0.4: + resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + '@types/estree': 1.0.5 + acorn: 8.11.3 + estree-walker: 3.0.3 + periscopic: 3.1.0 + dev: false + /collect-v8-coverage@1.0.2: resolution: {integrity: sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==} dev: false @@ -3805,6 +3987,18 @@ packages: which: 2.0.2 dev: false + /crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + dev: false + + /css-tree@2.3.1: + resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} + dependencies: + mdn-data: 2.0.30 + source-map-js: 1.0.2 + dev: false + /css.escape@1.5.1: resolution: {integrity: sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg==} dev: false @@ -3966,11 +4160,22 @@ packages: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} dev: false + /diff-match-patch@1.0.5: + resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} + dev: false + /diff-sequences@29.6.3: resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dev: false + /digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + dev: false + /dir-glob@3.0.1: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} @@ -4508,11 +4713,31 @@ packages: engines: {node: '>=4.0'} dev: false + /estree-walker@2.0.2: + resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} + dev: false + + /estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + dependencies: + '@types/estree': 1.0.5 + dev: false + /esutils@2.0.3: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} dev: false + /event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + dev: false + + /eventsource-parser@1.0.0: + resolution: {integrity: sha512-9jgfSCa3dmEme2ES3mPByGXfgZ87VbP97tng1G2nWwWx6bV2nYxm2AWCrbQjXToSe+yYlqaZNtxffR9IeQr95g==} + engines: {node: '>=14.18'} + dev: false + /execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -4647,6 +4872,10 @@ packages: signal-exit: 4.1.0 dev: false + /form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + dev: false + /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4656,6 +4885,14 @@ packages: mime-types: 2.1.35 dev: false + /formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + dev: false + /fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} dev: false @@ -4930,6 +5167,12 @@ packages: engines: {node: '>=10.17.0'} dev: false + /humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + dependencies: + ms: 2.1.3 + dev: false + /iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -5043,6 +5286,10 @@ packages: has-tostringtag: 1.0.2 dev: false + /is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + dev: false + /is-buffer@2.0.5: resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} engines: {node: '>=4'} @@ -5136,6 +5383,12 @@ packages: resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} dev: false + /is-reference@3.0.2: + resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} + dependencies: + '@types/estree': 1.0.5 + dev: false + /is-regex@1.1.4: resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} engines: {node: '>= 0.4'} @@ -5820,6 +6073,16 @@ packages: hasBin: true dev: false + /jsondiffpatch@0.6.0: + resolution: {integrity: sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + dependencies: + '@types/diff-match-patch': 1.0.36 + chalk: 5.3.0 + diff-match-patch: 1.0.5 + dev: false + /jsx-ast-utils@3.3.5: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} @@ -5879,6 +6142,10 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: false + /locate-character@3.0.0: + resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} + dev: false + /locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -5931,6 +6198,13 @@ packages: hasBin: true dev: false + /magic-string@0.30.8: + resolution: {integrity: sha512-ISQTe55T2ao7XtlAStud6qwYPZjE4GK1S/BeVPus4jrq6JuOnQ00YKQC581RWhR122W7msZV263KzVeLoqidyQ==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: false + /make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} @@ -5949,6 +6223,18 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false + /md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + dev: false + + /mdn-data@2.0.30: + resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + dev: false + /merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} dev: false @@ -6036,6 +6322,12 @@ packages: thenify-all: 1.6.0 dev: false + /nanoid@3.3.6: + resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: false + /nanoid@3.3.7: resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -6122,6 +6414,23 @@ packages: - babel-plugin-macros dev: false + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + dependencies: + whatwg-url: 5.0.0 + dev: false + /node-gyp-build@4.8.0: resolution: {integrity: sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og==} hasBin: true @@ -6254,6 +6563,23 @@ packages: mimic-fn: 2.1.0 dev: false + /openai@4.29.2: + resolution: {integrity: sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==} + hasBin: true + dependencies: + '@types/node': 18.19.24 + '@types/node-fetch': 2.6.11 + abort-controller: 3.0.0 + agentkeepalive: 4.5.0 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding + dev: false + /opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -6359,6 +6685,14 @@ packages: engines: {node: '>=8'} dev: false + /periscopic@3.1.0: + resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} + dependencies: + '@types/estree': 1.0.5 + estree-walker: 3.0.3 + is-reference: 3.0.2 + dev: false + /pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} @@ -6853,6 +7187,20 @@ packages: lru-cache: 6.0.0 dev: false + /seroval-plugins@1.0.5(seroval@1.0.5): + resolution: {integrity: sha512-8+pDC1vOedPXjKG7oz8o+iiHrtF2WswaMQJ7CKFpccvSYfrzmvKY9zOJWCg+881722wIHfwkdnRmiiDm9ym+zQ==} + engines: {node: '>=10'} + peerDependencies: + seroval: ^1.0 + dependencies: + seroval: 1.0.5 + dev: false + + /seroval@1.0.5: + resolution: {integrity: sha512-TM+Z11tHHvQVQKeNlOUonOWnsNM+2IBwZ4vwoi4j3zKzIpc5IDw8WPwCfcc8F17wy6cBcJGbZbFOR0UCuTZHQA==} + engines: {node: '>=10'} + dev: false + /server-only@0.0.1: resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} dev: false @@ -6927,6 +7275,25 @@ packages: engines: {node: '>=8'} dev: false + /solid-js@1.8.15: + resolution: {integrity: sha512-d0QP/efr3UVcwGgWVPveQQ0IHOH6iU7yUhc2piy8arNG8wxKmvUy1kFxyF8owpmfCWGB87usDKMaVnsNYZm+Vw==} + dependencies: + csstype: 3.1.3 + seroval: 1.0.5 + seroval-plugins: 1.0.5(seroval@1.0.5) + dev: false + + /solid-swr-store@0.10.7(solid-js@1.8.15)(swr-store@0.10.6): + resolution: {integrity: sha512-A6d68aJmRP471aWqKKPE2tpgOiR5fH4qXQNfKIec+Vap+MGQm3tvXlT8n0I8UgJSlNAsSAUuw2VTviH2h3Vv5g==} + engines: {node: '>=10'} + peerDependencies: + solid-js: ^1.2 + swr-store: ^0.10 + dependencies: + solid-js: 1.8.15 + swr-store: 0.10.6 + dev: false + /sonner@1.4.3(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-SArYlHbkjqRuLiR0iGY2ZSr09oOrxw081ZZkQPfXrs8aZQLIBOLOdzTYxGJB5yIZ7qL56UEPmrX1YqbODwG0Lw==} peerDependencies: @@ -6958,6 +7325,15 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: false + /sswr@2.0.0(svelte@4.2.12): + resolution: {integrity: sha512-mV0kkeBHcjcb0M5NqKtKVg/uTIYNlIIniyDfSGrSfxpEdM9C365jK0z55pl9K0xAkNTJi2OAOVFQpgMPUk+V0w==} + peerDependencies: + svelte: ^4.0.0 + dependencies: + svelte: 4.2.12 + swrev: 4.0.0 + dev: false + /stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -7145,6 +7521,54 @@ packages: engines: {node: '>= 0.4'} dev: false + /svelte@4.2.12: + resolution: {integrity: sha512-d8+wsh5TfPwqVzbm4/HCXC783/KPHV60NvwitJnyTA5lWn1elhXMNWhXGCJ7PwPa8qFUnyJNIyuIRt2mT0WMug==} + engines: {node: '>=16'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.22 + '@types/estree': 1.0.5 + acorn: 8.11.3 + aria-query: 5.3.0 + axobject-query: 4.0.0 + code-red: 1.0.4 + css-tree: 2.3.1 + estree-walker: 3.0.3 + is-reference: 3.0.2 + locate-character: 3.0.0 + magic-string: 0.30.8 + periscopic: 3.1.0 + dev: false + + /swr-store@0.10.6: + resolution: {integrity: sha512-xPjB1hARSiRaNNlUQvWSVrG5SirCjk2TmaUyzzvk69SZQan9hCJqw/5rG9iL7xElHU784GxRPISClq4488/XVw==} + engines: {node: '>=10'} + dependencies: + dequal: 2.0.3 + dev: false + + /swr@2.2.0(react@18.2.0): + resolution: {integrity: sha512-AjqHOv2lAhkuUdIiBu9xbuettzAzWXmCEcLONNKJRba87WAefz8Ca9d6ds/SzrPc235n1IxWYdhJ2zF3MNUaoQ==} + peerDependencies: + react: ^16.11.0 || ^17.0.0 || ^18.0.0 + dependencies: + react: 18.2.0 + use-sync-external-store: 1.2.0(react@18.2.0) + dev: false + + /swrev@4.0.0: + resolution: {integrity: sha512-LqVcOHSB4cPGgitD1riJ1Hh4vdmITOp+BkmfmXRh4hSF/t7EnS4iD+SOTmq7w5pPm/SiPeto4ADbKS6dHUDWFA==} + dev: false + + /swrv@1.0.4(vue@3.4.21): + resolution: {integrity: sha512-zjEkcP8Ywmj+xOJW3lIT65ciY/4AL4e/Or7Gj0MzU3zBJNMdJiT8geVZhINavnlHRMMCcJLHhraLTAiDOTmQ9g==} + peerDependencies: + vue: '>=3.2.26 < 4' + dependencies: + vue: 3.4.21(typescript@5.4.2) + dev: false + /symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} dev: false @@ -7242,6 +7666,10 @@ packages: url-parse: 1.5.10 dev: false + /tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + dev: false + /tr46@3.0.0: resolution: {integrity: sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==} engines: {node: '>=12'} @@ -7456,6 +7884,14 @@ packages: tslib: 2.6.2 dev: false + /use-sync-external-store@1.2.0(react@18.2.0): + resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + react: 18.2.0 + dev: false + /utf-8-validate@6.0.3: resolution: {integrity: sha512-uIuGf9TWQ/y+0Lp+KGZCMuJWc3N9BHA+l/UmHd/oUHwJJDeysyTRxNQVkbzsIWfGFbRe3OcgML/i0mvVRPOyDA==} engines: {node: '>=6.14.2'} @@ -7482,6 +7918,22 @@ packages: convert-source-map: 2.0.0 dev: false + /vue@3.4.21(typescript@5.4.2): + resolution: {integrity: sha512-5hjyV/jLEIKD/jYl4cavMcnzKwjMKohureP8ejn3hhEjwhWIhWeuzL2kJAjzl/WyVsgPY56Sy4Z40C3lVshxXA==} + peerDependencies: + typescript: '*' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@vue/compiler-dom': 3.4.21 + '@vue/compiler-sfc': 3.4.21 + '@vue/runtime-dom': 3.4.21 + '@vue/server-renderer': 3.4.21(vue@3.4.21) + '@vue/shared': 3.4.21 + typescript: 5.4.2 + dev: false + /w3c-xmlserializer@4.0.0: resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==} engines: {node: '>=14'} @@ -7495,6 +7947,20 @@ packages: makeerror: 1.0.12 dev: false + /web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + dev: false + + /web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + dev: false + + /webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + dev: false + /webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -7543,6 +8009,13 @@ packages: webidl-conversions: 7.0.0 dev: false + /whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + dev: false + /which-boxed-primitive@1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} dependencies: @@ -7725,3 +8198,15 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} dev: false + + /zod-to-json-schema@3.22.4(zod@3.22.4): + resolution: {integrity: sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ==} + peerDependencies: + zod: ^3.22.4 + dependencies: + zod: 3.22.4 + dev: false + + /zod@3.22.4: + resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} + dev: false diff --git a/src/components/CanvasBlurCapture.tsx b/src/components/CanvasBlurCapture.tsx index a4541e04..f72de914 100644 --- a/src/components/CanvasBlurCapture.tsx +++ b/src/components/CanvasBlurCapture.tsx @@ -6,6 +6,7 @@ const RETRY_DELAY = 2000; export default function CanvasBlurCapture({ imageUrl, + onLoad, onCapture, width, height, @@ -15,7 +16,8 @@ export default function CanvasBlurCapture({ quality = 0.9, }: { imageUrl: string - onCapture: (blurData: string) => void + onLoad?: (imageData: string) => void + onCapture: (imageData: string) => void width: number height: number hidden?: boolean @@ -44,7 +46,18 @@ export default function CanvasBlurCapture({ canvas.style.height = `${height}px`; const context = refCanvas.current?.getContext('2d'); if (context) { + // Draw scaled image context.scale(scale, scale); + context.drawImage( + refImage.current, + -edgeCompensation, + -edgeCompensation, + width + edgeCompensation * 2, + width * refImage.current.height / refImage.current.width + + edgeCompensation * 2, + ); + onLoad?.(canvas.toDataURL('image/jpeg', quality)); + // Draw blurred image context.filter = 'contrast(1.2) saturate(1.2) ' + `blur(${scale * 10}px)`; @@ -56,8 +69,8 @@ export default function CanvasBlurCapture({ width * refImage.current.height / refImage.current.width + edgeCompensation * 2, ); - refTimeouts.current.forEach(clearTimeout); onCapture(canvas.toDataURL('image/jpeg', quality)); + refTimeouts.current.forEach(clearTimeout); refShouldCapture.current = false; } else { console.error('Cannot get 2d context'); @@ -92,6 +105,7 @@ export default function CanvasBlurCapture({ }, [ imageUrl, onCapture, + onLoad, width, height, edgeCompensation, diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 6f4e4b76..ca61d471 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -49,6 +49,8 @@ export default function PhotoForm({ useState>(initialPhotoForm); const [formErrors, setFormErrors] = useState(getFormErrors(initialPhotoForm)); + const [imageData, setImageData] = + useState(); // Update form when EXIF data // is refreshed by parent @@ -116,6 +118,9 @@ export default function PhotoForm({ return (
+
{debugBlur && formData.blurData && diff --git a/src/services/openai.ts b/src/services/openai.ts new file mode 100644 index 00000000..f27880a9 --- /dev/null +++ b/src/services/openai.ts @@ -0,0 +1,37 @@ +'use server'; + +import OpenAI from 'openai'; +import { OpenAIStream, StreamingTextResponse } from 'ai'; + +const openai = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); + +const queryImage = async (imageBase64: string, query: string) => { + const response = await openai.chat.completions.create({ + model: 'gpt-4-vision-preview', + stream: true, + messages: [{ + 'role': 'user', + 'content': [ + { + 'type': 'text', + 'text': query, + }, { + 'type': 'image_url', + 'image_url': { + 'url': imageBase64, + }, + }, + ], + }], + }); + + const stream = OpenAIStream(response); + + return new StreamingTextResponse(stream); +}; + +export const tagImage = async (imageBase64: string) => + queryImage( + imageBase64, + 'Describe this image three or less comma-separated keywords', + ); From 137b718fb7aa7792b8006dec528f0aaf9afc7ba4 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 19:06:31 -0500 Subject: [PATCH 02/38] Create proof-of-concept AI-driven image description --- .vscode/settings.json | 5 +++-- src/photo/form/PhotoForm.tsx | 25 ++++++++++++++++++++++- src/services/openai.ts | 39 ++++++++++++++++++++++++------------ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 03182fcf..ddaa4892 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,6 +10,7 @@ "cloudflarestorage", "cmdk", "CredentialsSignin", + "datetime", "Eterna", "exif", "exifr", @@ -33,6 +34,7 @@ "Reala", "skippable", "sonner", + "Streamable", "thephotoblog", "trpc", "unnest", @@ -41,8 +43,7 @@ "WRHGZC", "wxyz", "zadd", - "zrange", - "datetime" + "zrange" ], "files.associations": { "*.css": "tailwindcss" diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index ca61d471..2fd5ad70 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -25,6 +25,9 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; +import { streamImageQuery } from '@/services/openai'; +import { readStreamableValue } from 'ai/rsc'; +import Spinner from '@/components/Spinner'; const THUMBNAIL_SIZE = 300; @@ -116,9 +119,22 @@ export default function PhotoForm({ } }, []); + const [aiTags, setAiTags] = useState(''); + const [isLoadingAi, setIsLoadingAi] = useState(false); + return (
-
@@ -152,6 +168,13 @@ export default function PhotoForm({ height={height} />}
+

+ AI RESPONSE: {aiTags} {isLoadingAi && <> + + + + } +

blur()} diff --git a/src/services/openai.ts b/src/services/openai.ts index f27880a9..0d91a18b 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -1,14 +1,16 @@ 'use server'; import OpenAI from 'openai'; -import { OpenAIStream, StreamingTextResponse } from 'ai'; +import { createStreamableValue, render } from 'ai/rsc'; -const openai = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); +const provider = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); -const queryImage = async (imageBase64: string, query: string) => { - const response = await openai.chat.completions.create({ +const streamImageQueryRaw = async (imageBase64: string, query: string) => { + const stream = createStreamableValue(''); + + render({ + provider, model: 'gpt-4-vision-preview', - stream: true, messages: [{ 'role': 'user', 'content': [ @@ -23,15 +25,26 @@ const queryImage = async (imageBase64: string, query: string) => { }, ], }], + text: ({ content, done }): any => { + if (done) { + stream.done(content); + } else { + stream.update(content); + } + }, }); - const stream = OpenAIStream(response); - - return new StreamingTextResponse(stream); + return stream.value; }; -export const tagImage = async (imageBase64: string) => - queryImage( - imageBase64, - 'Describe this image three or less comma-separated keywords', - ); +export type ImageQuery = 'title' | 'caption' | 'tags' | 'description'; + +export const IMAGE_QUERIES: Record = { + title: 'What is the title of this image?', + caption: 'What is the caption of this image?', + tags: 'Describe this image three or less comma-separated keywords', + description: 'Describe this image in detail', +}; + +export const streamImageQuery = (imageBase64: string, query: ImageQuery) => + streamImageQueryRaw(imageBase64, IMAGE_QUERIES[query]); From fdd392bf25b607ebde883b07a62557eea4e71034 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 20:07:56 -0500 Subject: [PATCH 03/38] Refactor AI server action code --- src/auth/index.ts | 2 +- src/photo/actions.ts | 33 +++++++++++++++++++++------------ src/photo/ai.ts | 13 +++++++++++++ src/services/openai.ts | 17 ++++------------- 4 files changed, 39 insertions(+), 26 deletions(-) create mode 100644 src/photo/ai.ts diff --git a/src/auth/index.ts b/src/auth/index.ts index 4ea60774..20ae70af 100644 --- a/src/auth/index.ts +++ b/src/auth/index.ts @@ -44,7 +44,7 @@ export const { }, }); -export const safelyRunServerAdminAction = async ( +export const safelyRunAdminServerAction = async ( callback: () => T, ): Promise => { const session = await auth(); diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 7b2033e7..759e2641 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -33,10 +33,11 @@ import { import { extractExifDataFromBlobPath } from './server'; import { TAG_FAVS, isTagFavs } from '@/tag'; import { convertPhotoToPhotoDbInsert } from '.'; -import { safelyRunServerAdminAction } from '@/auth'; +import { safelyRunAdminServerAction } from '@/auth'; +import { ImageQuery, streamImageQuery } from './ai'; export async function createPhotoAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const photo = convertFormDataToPhotoDbInsert(formData, true); const updatedUrl = await convertUploadToPhoto(photo.url, photo.id); @@ -52,7 +53,7 @@ export async function createPhotoAction(formData: FormData) { } export async function updatePhotoAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const photo = convertFormDataToPhotoDbInsert(formData); await sqlUpdatePhoto(photo); @@ -67,7 +68,7 @@ export async function toggleFavoritePhotoAction( photoId: string, shouldRedirect?: boolean, ) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const photo = await getPhoto(photoId); if (photo) { const { tags } = photo; @@ -88,7 +89,7 @@ export async function deletePhotoAction( photoUrl: string, shouldRedirect?: boolean, ) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { await sqlDeletePhoto(photoId).then(() => deleteStorageUrl(photoUrl)); revalidateAllKeysAndPaths(); if (shouldRedirect) { @@ -98,7 +99,7 @@ export async function deletePhotoAction( }; export async function deletePhotoFormAction(formData: FormData) { - return safelyRunServerAdminAction(async () => + return safelyRunAdminServerAction(async () => deletePhotoAction( formData.get('id') as string, formData.get('url') as string, @@ -107,7 +108,7 @@ export async function deletePhotoFormAction(formData: FormData) { }; export async function deletePhotoTagGloballyAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const tag = formData.get('tag') as string; await sqlDeletePhotoTagGlobally(tag); @@ -118,7 +119,7 @@ export async function deletePhotoTagGloballyAction(formData: FormData) { } export async function renamePhotoTagGloballyAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const tag = formData.get('tag') as string; const updatedTag = formData.get('updatedTag') as string; @@ -132,7 +133,7 @@ export async function renamePhotoTagGloballyAction(formData: FormData) { } export async function deleteBlobPhotoAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { await deleteStorageUrl(formData.get('url') as string); revalidateAdminPaths(); @@ -146,7 +147,7 @@ export async function deleteBlobPhotoAction(formData: FormData) { export async function getExifDataAction( photoFormPrevious: Partial, ): Promise> { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const { url } = photoFormPrevious; if (url) { const { photoFormExif } = await extractExifDataFromBlobPath(url); @@ -159,7 +160,7 @@ export async function getExifDataAction( } export async function syncPhotoExifDataAction(formData: FormData) { - return safelyRunServerAdminAction(async () => { + return safelyRunAdminServerAction(async () => { const photoId = formData.get('id') as string; if (photoId) { const photo = await getPhoto(photoId); @@ -179,5 +180,13 @@ export async function syncPhotoExifDataAction(formData: FormData) { } export async function syncCacheAction() { - return safelyRunServerAdminAction(revalidateAllKeysAndPaths); + return safelyRunAdminServerAction(revalidateAllKeysAndPaths); +} + +export async function streamImageQueryAction( + imageBase64: string, + query: ImageQuery, +) { + return safelyRunAdminServerAction(async () => + streamImageQuery(imageBase64, query)); } diff --git a/src/photo/ai.ts b/src/photo/ai.ts new file mode 100644 index 00000000..6e074de3 --- /dev/null +++ b/src/photo/ai.ts @@ -0,0 +1,13 @@ +import { streamOpenAiImageQuery } from '@/services/openai'; + +export type ImageQuery = 'title' | 'caption' | 'tags' | 'description'; + +export const IMAGE_QUERIES: Record = { + title: 'What is the title of this image?', + caption: 'What is the caption of this image?', + tags: 'Describe this image three or less comma-separated keywords', + description: 'Describe this image in detail', +}; + +export const streamImageQuery = (imageBase64: string, query: ImageQuery) => + streamOpenAiImageQuery(imageBase64, IMAGE_QUERIES[query]); diff --git a/src/services/openai.ts b/src/services/openai.ts index 0d91a18b..602d5a02 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -5,7 +5,10 @@ import { createStreamableValue, render } from 'ai/rsc'; const provider = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); -const streamImageQueryRaw = async (imageBase64: string, query: string) => { +export const streamOpenAiImageQuery = async ( + imageBase64: string, + query: string, +) => { const stream = createStreamableValue(''); render({ @@ -36,15 +39,3 @@ const streamImageQueryRaw = async (imageBase64: string, query: string) => { return stream.value; }; - -export type ImageQuery = 'title' | 'caption' | 'tags' | 'description'; - -export const IMAGE_QUERIES: Record = { - title: 'What is the title of this image?', - caption: 'What is the caption of this image?', - tags: 'Describe this image three or less comma-separated keywords', - description: 'Describe this image in detail', -}; - -export const streamImageQuery = (imageBase64: string, query: ImageQuery) => - streamImageQueryRaw(imageBase64, IMAGE_QUERIES[query]); From 0fcfa1b3c17fc5849b4abe6f25c4304a036abdb2 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 20:12:47 -0500 Subject: [PATCH 04/38] Fix AI action import --- src/photo/form/PhotoForm.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 2fd5ad70..321ab467 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -9,7 +9,11 @@ import { isFormValid, } from '.'; import FieldSetWithStatus from '@/components/FieldSetWithStatus'; -import { createPhotoAction, updatePhotoAction } from '../actions'; +import { + createPhotoAction, + streamImageQueryAction, + updatePhotoAction, +} from '../actions'; import SubmitButtonWithStatus from '@/components/SubmitButtonWithStatus'; import Link from 'next/link'; import { clsx } from 'clsx/lite'; @@ -25,7 +29,6 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; -import { streamImageQuery } from '@/services/openai'; import { readStreamableValue } from 'ai/rsc'; import Spinner from '@/components/Spinner'; @@ -126,7 +129,7 @@ export default function PhotoForm({
+
+ + + +
}

- AI RESPONSE: {aiTags} {isLoadingAi && <> + AI RESPONSE 01: {title} {isLoadingTitle && <> + + + + } +

+

+ AI RESPONSE 02: {aiTags.two} {isLoadingAi.two && <> + + + + } +

+

+ AI RESPONSE 01: {aiTags.three} {isLoadingAi.three && <> From dc7b0694ab0ef2b692708c5baa3f9e9dd267f792 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 22:22:00 -0500 Subject: [PATCH 06/38] Move all AI requests to useImageQuery() --- src/photo/form/PhotoForm.tsx | 74 +++++++++--------------------------- 1 file changed, 17 insertions(+), 57 deletions(-) diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 24bf6b20..6228d714 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -9,11 +9,7 @@ import { isFormValid, } from '.'; import FieldSetWithStatus from '@/components/FieldSetWithStatus'; -import { - createPhotoAction, - streamImageQueryAction, - updatePhotoAction, -} from '../actions'; +import { createPhotoAction, updatePhotoAction } from '../actions'; import SubmitButtonWithStatus from '@/components/SubmitButtonWithStatus'; import Link from 'next/link'; import { clsx } from 'clsx/lite'; @@ -29,7 +25,6 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; -import { readStreamableValue } from 'ai/rsc'; import Spinner from '@/components/Spinner'; import useImageQuery from '../ai/useImageQuery'; @@ -129,63 +124,28 @@ export default function PhotoForm({ isLoadingTitle, ] = useImageQuery(imageData, 'title'); - const [aiTags, setAiTags] = useState({ - two: '', - three: '', - }); - const [isLoadingAi, setIsLoadingAi] = useState({ - two: false, - three: false, - }); + const [ + requestTags, + tags, + isLoadingTags, + ] = useImageQuery(imageData, 'tags'); + + const [ + requestDescription, + description, + isLoadingDescription, + ] = useImageQuery(imageData, 'description'); return (

- - -
@@ -228,14 +188,14 @@ export default function PhotoForm({ }

- AI RESPONSE 02: {aiTags.two} {isLoadingAi.two && <> + AI RESPONSE 02: {tags} {isLoadingTags && <> }

- AI RESPONSE 01: {aiTags.three} {isLoadingAi.three && <> + AI RESPONSE 01: {description} {isLoadingDescription && <> From f39fa417b86bfd69e4c6f0da9d08d2328619e41c Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 23:43:00 -0500 Subject: [PATCH 07/38] Add error handling to AI text generation --- src/photo/ai/useImageQuery.ts | 21 ++++++++++++++------- src/photo/form/PhotoForm.tsx | 6 +++--- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/photo/ai/useImageQuery.ts b/src/photo/ai/useImageQuery.ts index 2893f930..51e8ec93 100644 --- a/src/photo/ai/useImageQuery.ts +++ b/src/photo/ai/useImageQuery.ts @@ -8,19 +8,25 @@ export default function useImageQuery( query: ImageQuery, ) { const [text, setText] = useState(''); + const [error, setError] = useState(); const [isLoading, setIsLoading] = useState(false); const request = useCallback(async () => { if (imageBase64) { setIsLoading(true); - const textStream = await streamImageQueryAction( - imageBase64 ?? '', - query, - ); - for await (const text of readStreamableValue(textStream)) { - setText(text ?? ''); + try { + const textStream = await streamImageQueryAction( + imageBase64 ?? '', + query, + ); + for await (const text of readStreamableValue(textStream)) { + setText(text ?? ''); + } + setIsLoading(false); + } catch (e) { + setError(e); + setIsLoading(false); } - setIsLoading(false); } }, [imageBase64, query]); @@ -28,5 +34,6 @@ export default function useImageQuery( request, text, isLoading, + error, ] as const; }; diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 6228d714..6b5b9ba5 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -181,21 +181,21 @@ export default function PhotoForm({ />}

- AI RESPONSE 01: {title} {isLoadingTitle && <> + ✨ TITLE: {title} {isLoadingTitle && <> }

- AI RESPONSE 02: {tags} {isLoadingTags && <> + ✨ TAGS: {tags} {isLoadingTags && <> }

- AI RESPONSE 01: {description} {isLoadingDescription && <> + ✨ DESCRIPTION: {description} {isLoadingDescription && <> From 83217a3905f0c9782c4e68c6947eb89cfd4cdc74 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Tue, 19 Mar 2024 23:47:07 -0500 Subject: [PATCH 08/38] Add descriptions of different lengths --- src/photo/ai/index.ts | 12 ++++++++++-- src/photo/form/PhotoForm.tsx | 36 +++++++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index b6fa7d07..a2ea7b8e 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -1,12 +1,20 @@ import { streamOpenAiImageQuery } from '@/services/openai'; -export type ImageQuery = 'title' | 'caption' | 'tags' | 'description'; +export type ImageQuery = + 'title' | + 'caption' | + 'tags' | + 'descriptionSmall' | + 'descriptionMedium' | + 'descriptionLarge'; export const IMAGE_QUERIES: Record = { title: 'Provide a short title for this image', caption: 'What is the caption of this image?', tags: 'Describe this image three or less comma-separated keywords', - description: 'Describe this image in detail', + descriptionSmall: 'Describe this image succinctly', + descriptionMedium: 'Describe this image', + descriptionLarge: 'Describe this image in detail', }; export const streamImageQuery = (imageBase64: string, query: ImageQuery) => diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 6b5b9ba5..5faac6b7 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -131,10 +131,16 @@ export default function PhotoForm({ ] = useImageQuery(imageData, 'tags'); const [ - requestDescription, - description, - isLoadingDescription, - ] = useImageQuery(imageData, 'description'); + requestDescriptionSmall, + descriptionSmall, + isLoadingDescriptionSmall, + ] = useImageQuery(imageData, 'descriptionSmall'); + + const [ + requestDescriptionLarge, + descriptionLarge, + isLoadingDescriptionLarge, + ] = useImageQuery(imageData, 'descriptionLarge'); return (

@@ -145,8 +151,17 @@ export default function PhotoForm({ - +
@@ -195,7 +210,14 @@ export default function PhotoForm({ }

- ✨ DESCRIPTION: {description} {isLoadingDescription && <> + ✨ DESCRIPTION (S): {descriptionSmall} {isLoadingDescriptionSmall && <> + + + + } +

+

+ ✨ DESCRIPTION (L): {descriptionLarge} {isLoadingDescriptionLarge && <> From 4f8313f0de37b9da69a22cb40a9574cb4416785a Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 08:43:00 -0500 Subject: [PATCH 09/38] Refine AI button behavior --- src/photo/form/PhotoForm.tsx | 45 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 5faac6b7..b9443e70 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -142,27 +142,34 @@ export default function PhotoForm({ isLoadingDescriptionLarge, ] = useImageQuery(imageData, 'descriptionLarge'); + const renderAiButton = ( + label: string, + onClick: () => void, + isLoading: boolean, + ) => + ; + return (

-
- - - - +
+ {renderAiButton('Title', requestTitle, isLoadingTitle)} + {renderAiButton('Tags', requestTags, isLoadingTags)} + {renderAiButton( + 'Description (S)', + requestDescriptionSmall, + isLoadingDescriptionSmall, + )} + {renderAiButton( + 'Description (L)', + requestDescriptionLarge, + isLoadingDescriptionLarge, + )}
Date: Wed, 20 Mar 2024 08:46:09 -0500 Subject: [PATCH 10/38] Add AI error handling --- src/photo/form/PhotoForm.tsx | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index b9443e70..f6bcf950 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -122,35 +122,43 @@ export default function PhotoForm({ requestTitle, title, isLoadingTitle, + errorTitle, ] = useImageQuery(imageData, 'title'); const [ requestTags, tags, isLoadingTags, + errorTags, ] = useImageQuery(imageData, 'tags'); const [ requestDescriptionSmall, descriptionSmall, isLoadingDescriptionSmall, + errorDescriptionSmall, ] = useImageQuery(imageData, 'descriptionSmall'); const [ requestDescriptionLarge, descriptionLarge, isLoadingDescriptionLarge, + errorDescriptionLarge, ] = useImageQuery(imageData, 'descriptionLarge'); const renderAiButton = ( label: string, onClick: () => void, isLoading: boolean, + error?: any, ) => ; @@ -158,17 +166,29 @@ export default function PhotoForm({ return (
- {renderAiButton('Title', requestTitle, isLoadingTitle)} - {renderAiButton('Tags', requestTags, isLoadingTags)} + {renderAiButton( + 'Title', + requestTitle, + isLoadingTitle, + errorTitle, + )} + {renderAiButton( + 'Tags', + requestTags, + isLoadingTags, + errorTags, + )} {renderAiButton( 'Description (S)', requestDescriptionSmall, isLoadingDescriptionSmall, + errorDescriptionSmall, )} {renderAiButton( 'Description (L)', requestDescriptionLarge, isLoadingDescriptionLarge, + errorDescriptionLarge, )}
From f3d036a546dd03426da394418c328355e8ddd551 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 10:23:20 -0500 Subject: [PATCH 11/38] Improve canvas error handling --- src/components/CanvasBlurCapture.tsx | 5 +++++ src/photo/form/PhotoForm.tsx | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/components/CanvasBlurCapture.tsx b/src/components/CanvasBlurCapture.tsx index f72de914..af8eea89 100644 --- a/src/components/CanvasBlurCapture.tsx +++ b/src/components/CanvasBlurCapture.tsx @@ -8,6 +8,7 @@ export default function CanvasBlurCapture({ imageUrl, onLoad, onCapture, + onError, width, height, hidden = true, @@ -18,6 +19,7 @@ export default function CanvasBlurCapture({ imageUrl: string onLoad?: (imageData: string) => void onCapture: (imageData: string) => void + onError?: (error: string) => void width: number height: number hidden?: boolean @@ -74,12 +76,14 @@ export default function CanvasBlurCapture({ refShouldCapture.current = false; } else { console.error('Cannot get 2d context'); + onError?.('Cannot get 2d context'); // Retry capture in case canvas is not available refTimeouts.current.push(setTimeout(capture, RETRY_DELAY)); } } else { // eslint-disable-next-line max-len console.error('Cannot generate blur data: canvas/image not ready'); + onError?.('Cannot generate blur data: canvas/image not ready'); // Retry capture in case canvas is not available refTimeouts.current.push(setTimeout(capture, RETRY_DELAY)); } @@ -106,6 +110,7 @@ export default function CanvasBlurCapture({ imageUrl, onCapture, onLoad, + onError, width, height, edgeCompensation, diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index f6bcf950..5ca7e128 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -51,6 +51,8 @@ export default function PhotoForm({ useState>(initialPhotoForm); const [formErrors, setFormErrors] = useState(getFormErrors(initialPhotoForm)); + const [blurError, setBlurError] = + useState(); const [imageData, setImageData] = useState(); @@ -165,6 +167,10 @@ export default function PhotoForm({ return (
+ {blurError && +
+ {blurError} +
}
{renderAiButton( 'Title', @@ -209,6 +215,7 @@ export default function PhotoForm({ height={height} onLoad={setImageData} onCapture={updateBlurData} + onError={setBlurError} /> {debugBlur && formData.blurData && Date: Wed, 20 Mar 2024 10:25:54 -0500 Subject: [PATCH 12/38] Bump next --- package.json | 2 +- pnpm-lock.yaml | 86 +++++++++++++++++++++++++------------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/package.json b/package.json index c6aa9c2d..5c86d0da 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "nanoid": "^5.0.6", - "next": "14.1.3", + "next": "14.1.4", "next-auth": "5.0.0-beta.15", "next-themes": "^0.3.0", "openai": "^4.29.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f45e6a7e..5c3c822e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,7 +49,7 @@ dependencies: version: 7.2.0(eslint@8.57.0)(typescript@5.4.2) '@vercel/analytics': specifier: ^1.2.2 - version: 1.2.2(next@14.1.3)(react@18.2.0) + version: 1.2.2(next@14.1.4)(react@18.2.0) '@vercel/blob': specifier: ^0.22.1 version: 0.22.1 @@ -58,7 +58,7 @@ dependencies: version: 0.7.2 '@vercel/speed-insights': specifier: ^1.0.10 - version: 1.0.10(next@14.1.3)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21) + version: 1.0.10(next@14.1.4)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21) ai: specifier: ^3.0.13 version: 3.0.13(react@18.2.0)(solid-js@1.8.15)(svelte@4.2.12)(vue@3.4.21)(zod@3.22.4) @@ -99,11 +99,11 @@ dependencies: specifier: ^5.0.6 version: 5.0.6 next: - specifier: 14.1.3 - version: 14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) + specifier: 14.1.4 + version: 14.1.4(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) next-auth: specifier: 5.0.0-beta.15 - version: 5.0.0-beta.15(next@14.1.3)(react@18.2.0) + version: 5.0.0-beta.15(next@14.1.4)(react@18.2.0) next-themes: specifier: ^0.3.0 version: 0.3.0(react-dom@18.2.0)(react@18.2.0) @@ -1557,8 +1557,8 @@ packages: - utf-8-validate dev: false - /@next/env@14.1.3: - resolution: {integrity: sha512-VhgXTvrgeBRxNPjyfBsDIMvgsKDxjlpw4IAUsHCX8Gjl1vtHUYRT3+xfQ/wwvLPDd/6kqfLqk9Pt4+7gysuCKQ==} + /@next/env@14.1.4: + resolution: {integrity: sha512-e7X7bbn3Z6DWnDi75UWn+REgAbLEqxI8Tq2pkFOFAMpWAWApz/YCUhtWMWn410h8Q2fYiYL7Yg5OlxMOCfFjJQ==} dev: false /@next/eslint-plugin-next@14.1.3: @@ -1567,8 +1567,8 @@ packages: glob: 10.3.10 dev: false - /@next/swc-darwin-arm64@14.1.3: - resolution: {integrity: sha512-LALu0yIBPRiG9ANrD5ncB3pjpO0Gli9ZLhxdOu6ZUNf3x1r3ea1rd9Q+4xxUkGrUXLqKVK9/lDkpYIJaCJ6AHQ==} + /@next/swc-darwin-arm64@14.1.4: + resolution: {integrity: sha512-ubmUkbmW65nIAOmoxT1IROZdmmJMmdYvXIe8211send9ZYJu+SqxSnJM4TrPj9wmL6g9Atvj0S/2cFmMSS99jg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] @@ -1576,8 +1576,8 @@ packages: dev: false optional: true - /@next/swc-darwin-x64@14.1.3: - resolution: {integrity: sha512-E/9WQeXxkqw2dfcn5UcjApFgUq73jqNKaE5bysDm58hEUdUGedVrnRhblhJM7HbCZNhtVl0j+6TXsK0PuzXTCg==} + /@next/swc-darwin-x64@14.1.4: + resolution: {integrity: sha512-b0Xo1ELj3u7IkZWAKcJPJEhBop117U78l70nfoQGo4xUSvv0PJSTaV4U9xQBLvZlnjsYkc8RwQN1HoH/oQmLlQ==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] @@ -1585,8 +1585,8 @@ packages: dev: false optional: true - /@next/swc-linux-arm64-gnu@14.1.3: - resolution: {integrity: sha512-USArX9B+3rZSXYLFvgy0NVWQgqh6LHWDmMt38O4lmiJNQcwazeI6xRvSsliDLKt+78KChVacNiwvOMbl6g6BBw==} + /@next/swc-linux-arm64-gnu@14.1.4: + resolution: {integrity: sha512-457G0hcLrdYA/u1O2XkRMsDKId5VKe3uKPvrKVOyuARa6nXrdhJOOYU9hkKKyQTMru1B8qEP78IAhf/1XnVqKA==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] @@ -1594,8 +1594,8 @@ packages: dev: false optional: true - /@next/swc-linux-arm64-musl@14.1.3: - resolution: {integrity: sha512-esk1RkRBLSIEp1qaQXv1+s6ZdYzuVCnDAZySpa62iFTMGTisCyNQmqyCTL9P+cLJ4N9FKCI3ojtSfsyPHJDQNw==} + /@next/swc-linux-arm64-musl@14.1.4: + resolution: {integrity: sha512-l/kMG+z6MB+fKA9KdtyprkTQ1ihlJcBh66cf0HvqGP+rXBbOXX0dpJatjZbHeunvEHoBBS69GYQG5ry78JMy3g==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] @@ -1603,8 +1603,8 @@ packages: dev: false optional: true - /@next/swc-linux-x64-gnu@14.1.3: - resolution: {integrity: sha512-8uOgRlYEYiKo0L8YGeS+3TudHVDWDjPVDUcST+z+dUzgBbTEwSSIaSgF/vkcC1T/iwl4QX9iuUyUdQEl0Kxalg==} + /@next/swc-linux-x64-gnu@14.1.4: + resolution: {integrity: sha512-BapIFZ3ZRnvQ1uWbmqEGJuPT9cgLwvKtxhK/L2t4QYO7l+/DxXuIGjvp1x8rvfa/x1FFSsipERZK70pewbtJtw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] @@ -1612,8 +1612,8 @@ packages: dev: false optional: true - /@next/swc-linux-x64-musl@14.1.3: - resolution: {integrity: sha512-DX2zqz05ziElLoxskgHasaJBREC5Y9TJcbR2LYqu4r7naff25B4iXkfXWfcp69uD75/0URmmoSgT8JclJtrBoQ==} + /@next/swc-linux-x64-musl@14.1.4: + resolution: {integrity: sha512-mqVxTwk4XuBl49qn2A5UmzFImoL1iLm0KQQwtdRJRKl21ylQwwGCxJtIYo2rbfkZHoSKlh/YgztY0qH3wG1xIg==} engines: {node: '>= 10'} cpu: [x64] os: [linux] @@ -1621,8 +1621,8 @@ packages: dev: false optional: true - /@next/swc-win32-arm64-msvc@14.1.3: - resolution: {integrity: sha512-HjssFsCdsD4GHstXSQxsi2l70F/5FsRTRQp8xNgmQs15SxUfUJRvSI9qKny/jLkY3gLgiCR3+6A7wzzK0DBlfA==} + /@next/swc-win32-arm64-msvc@14.1.4: + resolution: {integrity: sha512-xzxF4ErcumXjO2Pvg/wVGrtr9QQJLk3IyQX1ddAC/fi6/5jZCZ9xpuL9Tzc4KPWMFq8GGWFVDMshZOdHGdkvag==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] @@ -1630,8 +1630,8 @@ packages: dev: false optional: true - /@next/swc-win32-ia32-msvc@14.1.3: - resolution: {integrity: sha512-DRuxD5axfDM1/Ue4VahwSxl1O5rn61hX8/sF0HY8y0iCbpqdxw3rB3QasdHn/LJ6Wb2y5DoWzXcz3L1Cr+Thrw==} + /@next/swc-win32-ia32-msvc@14.1.4: + resolution: {integrity: sha512-WZiz8OdbkpRw6/IU/lredZWKKZopUMhcI2F+XiMAcPja0uZYdMTZQRoQ0WZcvinn9xZAidimE7tN9W5v9Yyfyw==} engines: {node: '>= 10'} cpu: [ia32] os: [win32] @@ -1639,8 +1639,8 @@ packages: dev: false optional: true - /@next/swc-win32-x64-msvc@14.1.3: - resolution: {integrity: sha512-uC2DaDoWH7h1P/aJ4Fok3Xiw6P0Lo4ez7NbowW2VGNXw/Xv6tOuLUcxhBYZxsSUJtpeknCi8/fvnSpyCFp4Rcg==} + /@next/swc-win32-x64-msvc@14.1.4: + resolution: {integrity: sha512-4Rto21sPfw555sZ/XNLqfxDUNeLhNYGO2dlPqsnuCg8N8a2a9u1ltqBOPQ4vj1Gf7eJC0W2hHG2eYUHuiXgY2w==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -3110,7 +3110,7 @@ packages: resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} dev: false - /@vercel/analytics@1.2.2(next@14.1.3)(react@18.2.0): + /@vercel/analytics@1.2.2(next@14.1.4)(react@18.2.0): resolution: {integrity: sha512-X0rctVWkQV1e5Y300ehVNqpOfSOufo7ieA5PIdna8yX/U7Vjz0GFsGf4qvAhxV02uQ2CVt7GYcrFfddXXK2Y4A==} peerDependencies: next: '>= 13' @@ -3121,7 +3121,7 @@ packages: react: optional: true dependencies: - next: 14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) + next: 14.1.4(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 server-only: 0.0.1 dev: false @@ -3146,7 +3146,7 @@ packages: ws: 8.14.2(bufferutil@4.0.8)(utf-8-validate@6.0.3) dev: false - /@vercel/speed-insights@1.0.10(next@14.1.3)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21): + /@vercel/speed-insights@1.0.10(next@14.1.4)(react@18.2.0)(svelte@4.2.12)(vue@3.4.21): resolution: {integrity: sha512-4uzdKB0RW6Ff2FkzshzjZ+RlJfLPxgm/00i0XXgxfMPhwnnsk92YgtqsxT9OcPLdJUyVU1DqFlSWWjIQMPkh0g==} requiresBuild: true peerDependencies: @@ -3170,7 +3170,7 @@ packages: vue-router: optional: true dependencies: - next: 14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) + next: 14.1.4(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 svelte: 4.2.12 vue: 3.4.21(typescript@5.4.2) @@ -6344,7 +6344,7 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: false - /next-auth@5.0.0-beta.15(next@14.1.3)(react@18.2.0): + /next-auth@5.0.0-beta.15(next@14.1.4)(react@18.2.0): resolution: {integrity: sha512-UQggNq8CDu3/w8CYkihKLLnRPNXel98K0j7mtjj9a6XTNYo4Hni8xg/2h1YhElW6vXE8mgtvmH11rU8NKw86jQ==} peerDependencies: '@simplewebauthn/browser': ^9.0.1 @@ -6361,7 +6361,7 @@ packages: optional: true dependencies: '@auth/core': 0.28.0 - next: 14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) + next: 14.1.4(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 dev: false @@ -6375,8 +6375,8 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false - /next@14.1.3(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-oexgMV2MapI0UIWiXKkixF8J8ORxpy64OuJ/J9oVUmIthXOUCcuVEZX+dtpgq7wIfIqtBwQsKEDXejcjTsan9g==} + /next@14.1.4(@babel/core@7.23.9)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-1WTaXeSrUwlz/XcnhGTY7+8eiaFvdet5z9u3V2jb+Ek1vFo0VhHKSAIJvDWfQpttWjnyw14kBeq28TPq7bTeEQ==} engines: {node: '>=18.17.0'} hasBin: true peerDependencies: @@ -6390,7 +6390,7 @@ packages: sass: optional: true dependencies: - '@next/env': 14.1.3 + '@next/env': 14.1.4 '@swc/helpers': 0.5.2 busboy: 1.6.0 caniuse-lite: 1.0.30001591 @@ -6400,15 +6400,15 @@ packages: react-dom: 18.2.0(react@18.2.0) styled-jsx: 5.1.1(@babel/core@7.23.9)(react@18.2.0) optionalDependencies: - '@next/swc-darwin-arm64': 14.1.3 - '@next/swc-darwin-x64': 14.1.3 - '@next/swc-linux-arm64-gnu': 14.1.3 - '@next/swc-linux-arm64-musl': 14.1.3 - '@next/swc-linux-x64-gnu': 14.1.3 - '@next/swc-linux-x64-musl': 14.1.3 - '@next/swc-win32-arm64-msvc': 14.1.3 - '@next/swc-win32-ia32-msvc': 14.1.3 - '@next/swc-win32-x64-msvc': 14.1.3 + '@next/swc-darwin-arm64': 14.1.4 + '@next/swc-darwin-x64': 14.1.4 + '@next/swc-linux-arm64-gnu': 14.1.4 + '@next/swc-linux-arm64-musl': 14.1.4 + '@next/swc-linux-x64-gnu': 14.1.4 + '@next/swc-linux-x64-musl': 14.1.4 + '@next/swc-win32-arm64-msvc': 14.1.4 + '@next/swc-win32-ia32-msvc': 14.1.4 + '@next/swc-win32-x64-msvc': 14.1.4 transitivePeerDependencies: - '@babel/core' - babel-plugin-macros From 6d9f207cdfdeca67cce52c71796e31065a9fb619 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 10:28:58 -0500 Subject: [PATCH 13/38] Add granular ai loading spinners --- src/photo/form/PhotoForm.tsx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 5ca7e128..6a98bf41 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -158,11 +158,19 @@ export default function PhotoForm({ onClick={onClick} disabled={!imageData || isLoading} className={clsx( + 'flex gap-2 items-center justify-center', 'disabled:opacity-50 text-sm px-2.5 min-h-0 py-1.5', Boolean(error) && 'error text-error', )} > - {label} ✨ + + {label} + + + {isLoading + ? + : <>✨} + ; return ( From a351999e373775308491981417100e618952bc2a Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 13:18:10 -0500 Subject: [PATCH 14/38] Fine-tune AI text generation --- src/components/CanvasBlurCapture.tsx | 10 ++- src/photo/ai/index.ts | 16 +++- src/photo/ai/useImageQuery.ts | 7 +- src/photo/form/PhotoForm.tsx | 106 +++++++++++++++++++-------- 4 files changed, 99 insertions(+), 40 deletions(-) diff --git a/src/components/CanvasBlurCapture.tsx b/src/components/CanvasBlurCapture.tsx index af8eea89..cce1ed31 100644 --- a/src/components/CanvasBlurCapture.tsx +++ b/src/components/CanvasBlurCapture.tsx @@ -72,18 +72,20 @@ export default function CanvasBlurCapture({ edgeCompensation * 2, ); onCapture(canvas.toDataURL('image/jpeg', quality)); + onError?.(''); refTimeouts.current.forEach(clearTimeout); refShouldCapture.current = false; } else { - console.error('Cannot get 2d context'); - onError?.('Cannot get 2d context'); + console.error('Cannot get 2d context ... retrying'); + onError?.('Cannot get 2d context ... retrying'); // Retry capture in case canvas is not available refTimeouts.current.push(setTimeout(capture, RETRY_DELAY)); } } else { // eslint-disable-next-line max-len - console.error('Cannot generate blur data: canvas/image not ready'); - onError?.('Cannot generate blur data: canvas/image not ready'); + console.error('Cannot generate blur data: canvas/image not ready ... retrying'); + // eslint-disable-next-line max-len + onError?.('Cannot generate blur data: canvas/image not ready ... retrying'); // Retry capture in case canvas is not available refTimeouts.current.push(setTimeout(capture, RETRY_DELAY)); } diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index a2ea7b8e..b8375008 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -6,15 +6,23 @@ export type ImageQuery = 'tags' | 'descriptionSmall' | 'descriptionMedium' | - 'descriptionLarge'; + 'descriptionLarge' | + 'rich' | + 'semantic'; export const IMAGE_QUERIES: Record = { - title: 'Provide a short title for this image', - caption: 'What is the caption of this image?', - tags: 'Describe this image three or less comma-separated keywords', + // title: 'Provide a short title for this image', + title: 'Provide a short title for this image in 3 words or less', + caption: 'What is a pithy caption for this image in 8 words or less?', + // eslint-disable-next-line max-len + tags: 'Describe this image three or less comma-separated keywords with no adjective or adverbs', descriptionSmall: 'Describe this image succinctly', descriptionMedium: 'Describe this image', descriptionLarge: 'Describe this image in detail', + // eslint-disable-next-line max-len + rich: 'What is a short title and pithy caption of 8 words or less for this image?', + // eslint-disable-next-line max-len + semantic: 'List up to 5 things in this image without description as a comma-separated list', }; export const streamImageQuery = (imageBase64: string, query: ImageQuery) => diff --git a/src/photo/ai/useImageQuery.ts b/src/photo/ai/useImageQuery.ts index 51e8ec93..9f0079eb 100644 --- a/src/photo/ai/useImageQuery.ts +++ b/src/photo/ai/useImageQuery.ts @@ -16,7 +16,7 @@ export default function useImageQuery( setIsLoading(true); try { const textStream = await streamImageQueryAction( - imageBase64 ?? '', + imageBase64, query, ); for await (const text of readStreamableValue(textStream)) { @@ -30,9 +30,12 @@ export default function useImageQuery( } }, [imageBase64, query]); + // Withhold streaming text if it's a null response + const isTextError = text.toLocaleLowerCase().startsWith('sorry'); + return [ request, - text, + isTextError ? '' : text, isLoading, error, ] as const; diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 6a98bf41..e7068c69 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -120,12 +120,19 @@ export default function PhotoForm({ } }, []); - const [ - requestTitle, - title, - isLoadingTitle, - errorTitle, - ] = useImageQuery(imageData, 'title'); + // const [ + // requestTitle, + // title, + // isLoadingTitle, + // errorTitle, + // ] = useImageQuery(imageData, 'title'); + + // const [ + // requestCaption, + // caption, + // isLoadingCaption, + // errorCaption, + // ] = useImageQuery(imageData, 'caption'); const [ requestTags, @@ -135,18 +142,25 @@ export default function PhotoForm({ ] = useImageQuery(imageData, 'tags'); const [ - requestDescriptionSmall, - descriptionSmall, - isLoadingDescriptionSmall, - errorDescriptionSmall, - ] = useImageQuery(imageData, 'descriptionSmall'); + requestRich, + rich, + isLoadingRich, + errorRich, + ] = useImageQuery(imageData, 'rich'); + + // const [ + // requestDescriptionSmall, + // descriptionSmall, + // isLoadingDescriptionSmall, + // errorDescriptionSmall, + // ] = useImageQuery(imageData, 'descriptionSmall'); const [ - requestDescriptionLarge, - descriptionLarge, - isLoadingDescriptionLarge, - errorDescriptionLarge, - ] = useImageQuery(imageData, 'descriptionLarge'); + requestSemantic, + semantic, + isLoadingSemantic, + errorSemantic, + ] = useImageQuery(imageData, 'semantic'); const renderAiButton = ( label: string, @@ -180,12 +194,30 @@ export default function PhotoForm({ {blurError}
}
- {renderAiButton( + {/* {renderAiButton( 'Title', requestTitle, isLoadingTitle, errorTitle, )} + {renderAiButton( + 'Caption', + requestCaption, + isLoadingCaption, + errorCaption, + )} + {renderAiButton( + 'Tags', + requestTags, + isLoadingTags, + errorTags, + )} */} + {renderAiButton( + 'Rich', + requestRich, + isLoadingRich, + errorRich, + )} {renderAiButton( 'Tags', requestTags, @@ -193,17 +225,17 @@ export default function PhotoForm({ errorTags, )} {renderAiButton( - 'Description (S)', + 'Semantic', + requestSemantic, + isLoadingSemantic, + errorSemantic, + )} + {/* {renderAiButton( + 'Description', requestDescriptionSmall, isLoadingDescriptionSmall, errorDescriptionSmall, - )} - {renderAiButton( - 'Description (L)', - requestDescriptionLarge, - isLoadingDescriptionLarge, - errorDescriptionLarge, - )} + )} */}
}
-

+ {/*

✨ TITLE: {title} {isLoadingTitle && <> }

+

+ ✨ CAPTION: {caption} {isLoadingCaption && <> + + + + } +

*/} +

+ ✨ RICH: {rich} {isLoadingRich && <> + + + + } +

✨ TAGS: {tags} {isLoadingTags && <> @@ -252,19 +298,19 @@ export default function PhotoForm({ }

- ✨ DESCRIPTION (S): {descriptionSmall} {isLoadingDescriptionSmall && <> + ✨ SEMANTIC: {semantic} {isLoadingSemantic && <> }

-

- ✨ DESCRIPTION (L): {descriptionLarge} {isLoadingDescriptionLarge && <> + {/*

+ ✨ DESCRIPTION: {descriptionSmall} {isLoadingDescriptionSmall && <> } -

+

*/} blur()} From f7aa65101d0ed7e7785b5b054eba485146297402 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 15:31:28 -0500 Subject: [PATCH 15/38] Document AI text generation features --- README.md | 5 +++- src/app/admin/photos/[photoId]/edit/page.tsx | 9 ++++++- src/app/admin/uploads/[uploadPath]/page.tsx | 14 ++++++++--- src/photo/PhotoEditPageClient.tsx | 5 +++- src/photo/UploadPageClient.tsx | 3 +++ src/photo/form/PhotoForm.tsx | 5 +++- src/photo/form/index.ts | 10 +++++--- src/site/SiteChecklistClient.tsx | 26 +++++++++++++++++--- src/site/config.ts | 3 +++ 9 files changed, 66 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1a462320..456ddc6d 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,14 @@ https://photos.sambecker.com Features - +- Built-in auth - Photo upload with EXIF extraction - Organize photos by tag and camera model - Infinite scroll -- Built-in auth - Light/dark mode +- CMD-K menu with photo search - Automatic OG image generation +- Experimental support for AI-generated descriptions - Support for Fujifilm simulations OG Image Preview @@ -71,6 +73,7 @@ Installation - `NEXT_PUBLIC_PRO_MODE = 1` enables higher quality image storage for jpgs (will result in increased storage usage) - `NEXT_PUBLIC_BLUR_DISABLED = 1` prevents image blur data being stored and displayed (potentially useful for limiting Postgres usage) - `NEXT_PUBLIC_GEO_PRIVACY = 1` disables collection/display of location-based data +- `OPENAI_SECRET_KEY = [Your Key]` enables experimental support for AI-generated text descriptions - `NEXT_PUBLIC_IGNORE_PRIORITY_ORDER = 1` prevents `priority_order` field affecting photo order - `NEXT_PUBLIC_PUBLIC_API = 1` enables public API available at `/api` - `NEXT_PUBLIC_HIDE_REPO_LINK = 1` removes footer link to repo diff --git a/src/app/admin/photos/[photoId]/edit/page.tsx b/src/app/admin/photos/[photoId]/edit/page.tsx index 2a5c2d6c..767ba55b 100644 --- a/src/app/admin/photos/[photoId]/edit/page.tsx +++ b/src/app/admin/photos/[photoId]/edit/page.tsx @@ -2,6 +2,7 @@ import { redirect } from 'next/navigation'; import { getPhotoNoStore, getUniqueTagsCached } from '@/photo/cache'; import { PATH_ADMIN } from '@/site/paths'; import PhotoEditPageClient from '@/photo/PhotoEditPageClient'; +import { AI_TEXT_GENERATION_ENABLED } from '@/site/config'; export default async function PhotoEditPage({ params: { photoId }, @@ -14,7 +15,13 @@ export default async function PhotoEditPage({ const uniqueTags = await getUniqueTagsCached(); + const aiTextGeneration = AI_TEXT_GENERATION_ENABLED; + return ( - + ); }; diff --git a/src/app/admin/uploads/[uploadPath]/page.tsx b/src/app/admin/uploads/[uploadPath]/page.tsx index b59ccc45..d075631c 100644 --- a/src/app/admin/uploads/[uploadPath]/page.tsx +++ b/src/app/admin/uploads/[uploadPath]/page.tsx @@ -3,6 +3,7 @@ import { extractExifDataFromBlobPath } from '@/photo/server'; import { redirect } from 'next/navigation'; import { getUniqueTagsCached } from '@/photo/cache'; import UploadPageClient from '@/photo/UploadPageClient'; +import { AI_TEXT_GENERATION_ENABLED } from '@/site/config'; interface Params { params: { uploadPath: string } @@ -14,11 +15,18 @@ export default async function UploadPage({ params: { uploadPath } }: Params) { photoFormExif, } = await extractExifDataFromBlobPath(uploadPath); - const uniqueTags = await getUniqueTagsCached(); - if (!photoFormExif) { redirect(PATH_ADMIN); } + const uniqueTags = await getUniqueTagsCached(); + + const aiTextGeneration = AI_TEXT_GENERATION_ENABLED; + return ( - + ); }; diff --git a/src/photo/PhotoEditPageClient.tsx b/src/photo/PhotoEditPageClient.tsx index 6be696f5..a26c054d 100644 --- a/src/photo/PhotoEditPageClient.tsx +++ b/src/photo/PhotoEditPageClient.tsx @@ -16,9 +16,11 @@ import { useState } from 'react'; export default function PhotoEditPageClient({ photo, uniqueTags, + aiTextGeneration, }: { photo: Photo - uniqueTags?: Tags + uniqueTags: Tags + aiTextGeneration: boolean }) { const seedExifData = { url: photo.url }; @@ -62,6 +64,7 @@ export default function PhotoEditPageClient({ ? updatedExifData : undefined} uniqueTags={uniqueTags} + aiTextGeneration={aiTextGeneration} onTitleChange={setUpdatedTitle} onFormStatusChange={setIsPending} /> diff --git a/src/photo/UploadPageClient.tsx b/src/photo/UploadPageClient.tsx index b37a4d39..be755510 100644 --- a/src/photo/UploadPageClient.tsx +++ b/src/photo/UploadPageClient.tsx @@ -11,10 +11,12 @@ export default function UploadPageClient({ blobId, photoFormExif, uniqueTags, + aiTextGeneration, }: { blobId?: string photoFormExif: Partial uniqueTags: Tags + aiTextGeneration: boolean }) { const [pending, setIsPending] = useState(false); const [updatedTitle, setUpdatedTitle] = useState(''); @@ -31,6 +33,7 @@ export default function UploadPageClient({ diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index e7068c69..0ff11a6e 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -35,6 +35,7 @@ export default function PhotoForm({ updatedExifData, type = 'create', uniqueTags, + aiTextGeneration, debugBlur, onTitleChange, onFormStatusChange, @@ -43,6 +44,7 @@ export default function PhotoForm({ updatedExifData?: Partial type?: 'create' | 'edit' uniqueTags?: Tags + aiTextGeneration?: boolean debugBlur?: boolean onTitleChange?: (updatedTitle: string) => void onFormStatusChange?: (pending: boolean) => void @@ -322,7 +324,8 @@ export default function PhotoForm({ value: tag, annotation: formatCount(count), annotationAria: formatCountDescriptive(count, 'tagged'), - })) + })), + aiTextGeneration, ) .map(([key, { label, diff --git a/src/photo/form/index.ts b/src/photo/form/index.ts index 078e36c6..39ec8661 100644 --- a/src/photo/form/index.ts +++ b/src/photo/form/index.ts @@ -13,7 +13,10 @@ import { MAKE_FUJIFILM, } from '@/vendors/fujifilm'; import { FilmSimulation } from '@/simulation'; -import { BLUR_ENABLED, GEO_PRIVACY_ENABLED } from '@/site/config'; +import { + BLUR_ENABLED, + GEO_PRIVACY_ENABLED, +} from '@/site/config'; import { TAG_FAVS, doesTagsStringIncludeFavs } from '@/tag'; type VirtualFields = 'favorite'; @@ -55,7 +58,8 @@ const STRING_MAX_LENGTH_SHORT = 255; const STRING_MAX_LENGTH_LONG = 1000; const FORM_METADATA = ( - tagOptions?: AnnotatedTag[] + tagOptions?: AnnotatedTag[], + aiTextGeneration?: boolean, ): Record => ({ title: { label: 'title', @@ -72,7 +76,7 @@ const FORM_METADATA = ( label: 'semantic description', capitalize: true, validateStringMaxLength: STRING_MAX_LENGTH_LONG, - hide: true, + hide: !aiTextGeneration, }, tags: { label: 'tags', diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 7b08db87..0d8febec 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -40,6 +40,7 @@ export default function SiteChecklistClient({ isBlurEnabled, isGeoPrivacyEnabled, isPriorityOrderEnabled, + isAiTextGenerationEnabled, isPublicApiEnabled, isOgTextBottomAligned, gridAspectRatio, @@ -92,10 +93,16 @@ export default function SiteChecklistClient({ }} />; - const renderEnvVar = (variable: string) => + const renderEnvVar = ( + variable: string, + minimal?: boolean, + ) =>
`{variable}` - {renderCopyButton(variable, variable, true)} + {!minimal && renderCopyButton(variable, variable, true)}
; const renderEnvVars = (variables: string[]) =>
- {variables.map(renderEnvVar)} + {variables.map(envVar => renderEnvVar(envVar))}
; const renderSubStatus = ( @@ -294,6 +301,17 @@ export default function SiteChecklistClient({ collection/display of location-based data {renderEnvVars(['NEXT_PUBLIC_GEO_PRIVACY'])} + + Store your OpenAI secret key in order to add experimental support + for AI-generated text descriptions and enable an invisible field + called {'"Semantic Description"'} used to support CMD-K search + {renderEnvVars(['OPENAI_SECRET_KEY'])} + Date: Wed, 20 Mar 2024 15:37:17 -0500 Subject: [PATCH 16/38] Add experimental badge to AI-generated feature --- src/components/ChecklistRow.tsx | 17 ++++++++++++++++- src/site/SiteChecklistClient.tsx | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/components/ChecklistRow.tsx b/src/components/ChecklistRow.tsx index 10b91f5a..db6662fd 100644 --- a/src/components/ChecklistRow.tsx +++ b/src/components/ChecklistRow.tsx @@ -7,12 +7,14 @@ export default function ChecklistRow({ status, isPending, optional, + experimental, children, }: { title: string status: boolean isPending: boolean optional?: boolean + experimental?: boolean children: ReactNode }) { return ( @@ -25,8 +27,21 @@ export default function ChecklistRow({ loading={isPending} />
-
+
{title} + {experimental && + + Experimental + }
{children} diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 0d8febec..1fb9ec0d 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -305,6 +305,7 @@ export default function SiteChecklistClient({ title="AI-generated Text" status={isAiTextGenerationEnabled} isPending={isPendingPage} + experimental optional > Store your OpenAI secret key in order to add experimental support From 786378e4a50be6962d50e9abcd0dade65b50e072 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 18:57:19 -0500 Subject: [PATCH 17/38] Add AI rate limiting and safety documentation --- .vscode/settings.json | 4 +++ README.md | 17 ++++++++-- package.json | 2 ++ pnpm-lock.yaml | 42 ++++++++++++++++++++++++ src/services/openai.ts | 21 ++++++++++++ src/services/vercel-kv.ts | 10 ++++++ src/site/SiteChecklistClient.tsx | 56 ++++++++++++++++++++++---------- src/site/config.ts | 18 +++++++--- 8 files changed, 147 insertions(+), 23 deletions(-) create mode 100644 src/services/vercel-kv.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index ddaa4892..f3ea8edd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,6 +24,7 @@ "jpgs", "Lightbox", "Makernote", + "mitigations", "nanoids", "nextjs", "parameterizes", @@ -31,6 +32,8 @@ "Provia", "qaub", "QRSTUVWXYZ", + "ratelimit", + "ratelimiter", "Reala", "skippable", "sonner", @@ -38,6 +41,7 @@ "thephotoblog", "trpc", "unnest", + "upstash", "UsKSGcbt", "Velvia", "WRHGZC", diff --git a/README.md b/README.md index 456ddc6d..7d84d313 100644 --- a/README.md +++ b/README.md @@ -68,12 +68,25 @@ Installation 2. Click "Speed Insights" tab 3. Follow "Enable Speed Insights" instructions (`@vercel/speed-insights` already included) -### 7. Optional configuration +### 7. Add experimental AI text generation + +_⚠️ READ BEFORE PROCEEDING_ +- _Usage of this feature will result in fees from OpenAI._ +- _When enabling AI text generation, follow all recommended mitigations in order to avoid unexpected charges and attacks._ +- _Make sure your OpenAI secret key is not prefixed with NEXT_PUBLIC._ + +1. Setup OpenAI + - If you don't already have one, create an [OpenAI](https://openai.com) account + - Generate an API key and store as `OPENAI_SECRET_KEY` + - Setup usage limits to avoid unexpected charges (_recommended_) +2. Add rate limiting (_recommended_) + - As an additional precaution, create a [Vercel KV](https://vercel.com/docs/storage/vercel-kv/quickstart#create-a-kv-database) store and link it to your project in order to enable rate limiting + +### 8. Optional configuration - `NEXT_PUBLIC_PRO_MODE = 1` enables higher quality image storage for jpgs (will result in increased storage usage) - `NEXT_PUBLIC_BLUR_DISABLED = 1` prevents image blur data being stored and displayed (potentially useful for limiting Postgres usage) - `NEXT_PUBLIC_GEO_PRIVACY = 1` disables collection/display of location-based data -- `OPENAI_SECRET_KEY = [Your Key]` enables experimental support for AI-generated text descriptions - `NEXT_PUBLIC_IGNORE_PRIORITY_ORDER = 1` prevents `priority_order` field affecting photo order - `NEXT_PUBLIC_PUBLIC_API = 1` enables public API available at `/api` - `NEXT_PUBLIC_HIDE_REPO_LINK = 1` removes footer link to repo diff --git a/package.json b/package.json index 5c86d0da..73431121 100644 --- a/package.json +++ b/package.json @@ -23,8 +23,10 @@ "@types/react-dom": "18.2.21", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", + "@upstash/ratelimit": "^1.0.1", "@vercel/analytics": "^1.2.2", "@vercel/blob": "^0.22.1", + "@vercel/kv": "^1.0.1", "@vercel/postgres": "0.7.2", "@vercel/speed-insights": "^1.0.10", "ai": "^3.0.13", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c3c822e..70a1fae3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,12 +47,18 @@ dependencies: '@typescript-eslint/parser': specifier: ^7.2.0 version: 7.2.0(eslint@8.57.0)(typescript@5.4.2) + '@upstash/ratelimit': + specifier: ^1.0.1 + version: 1.0.1 '@vercel/analytics': specifier: ^1.2.2 version: 1.2.2(next@14.1.4)(react@18.2.0) '@vercel/blob': specifier: ^0.22.1 version: 0.22.1 + '@vercel/kv': + specifier: ^1.0.1 + version: 1.0.1 '@vercel/postgres': specifier: 0.7.2 version: 0.7.2 @@ -3110,6 +3116,31 @@ packages: resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} dev: false + /@upstash/core-analytics@0.0.7: + resolution: {integrity: sha512-lC2j5efqb1haX/fpTGaPUx1rue1WUkOZBVHDzCB7eMIVsRdFFp4xiHtyH/G9omiR1zj39fU5SCTWFiKJH3KOpw==} + engines: {node: '>=16.0.0'} + dependencies: + '@upstash/redis': 1.28.4 + dev: false + + /@upstash/ratelimit@1.0.1: + resolution: {integrity: sha512-G9LZ7idhlkuYknbUngCB3qzd7QnkK1xDkFG5jRtEJZuOUS5UKJ0UTKbhalCtp39eX2wu2Ubv8W7HCeaJQOWM0A==} + dependencies: + '@upstash/core-analytics': 0.0.7 + dev: false + + /@upstash/redis@1.25.1: + resolution: {integrity: sha512-ACj0GhJ4qrQyBshwFgPod6XufVEfKX2wcaihsEvSdLYnY+m+pa13kGt1RXm/yTHKf4TQi/Dy2A8z/y6WUEOmlg==} + dependencies: + crypto-js: 4.2.0 + dev: false + + /@upstash/redis@1.28.4: + resolution: {integrity: sha512-UalkSAny/dz1m8giEhD3Y5ru1o+CPHI32wFyS3MyzDzj2TRvEN+lTw+mPwi20ojk0H2gs8TBW3qsrvwuLLy+pA==} + dependencies: + crypto-js: 4.2.0 + dev: false + /@vercel/analytics@1.2.2(next@14.1.4)(react@18.2.0): resolution: {integrity: sha512-X0rctVWkQV1e5Y300ehVNqpOfSOufo7ieA5PIdna8yX/U7Vjz0GFsGf4qvAhxV02uQ2CVt7GYcrFfddXXK2Y4A==} peerDependencies: @@ -3136,6 +3167,13 @@ packages: undici: 5.28.3 dev: false + /@vercel/kv@1.0.1: + resolution: {integrity: sha512-uTKddsqVYS2GRAM/QMNNXCTuw9N742mLoGRXoNDcyECaxEXvIHG0dEY+ZnYISV4Vz534VwJO+64fd9XeSggSKw==} + engines: {node: '>=14.6'} + dependencies: + '@upstash/redis': 1.25.1 + dev: false + /@vercel/postgres@0.7.2: resolution: {integrity: sha512-IqR/ZAvoPGcPaXl9eWWB5KaA+w/81RzZa/18P4izQRHpNBkTGt9HwGfYi9+wut5UgxNq4QSX9A7HIQR6QDvX2Q==} engines: {node: '>=14.6'} @@ -3991,6 +4029,10 @@ packages: resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} dev: false + /crypto-js@4.2.0: + resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + dev: false + /css-tree@2.3.1: resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} diff --git a/src/services/openai.ts b/src/services/openai.ts index 602d5a02..4ef1661b 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -2,13 +2,34 @@ import OpenAI from 'openai'; import { createStreamableValue, render } from 'ai/rsc'; +import { kv } from '@/services/vercel-kv'; +import { Ratelimit } from '@upstash/ratelimit'; + +const RATE_LIMIT_IDENTIFIER = 'openai-image-query'; +const RATE_LIMIT_MAX_QUERIES_PER_HOUR = 100; const provider = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); +// Allows 100 requests per hour +const ratelimit = kv + ? new Ratelimit({ + redis: kv, + limiter: Ratelimit.slidingWindow(RATE_LIMIT_MAX_QUERIES_PER_HOUR, '1h'), + }) + : undefined; + export const streamOpenAiImageQuery = async ( imageBase64: string, query: string, ) => { + if (ratelimit) { + const { success } = await ratelimit.limit(RATE_LIMIT_IDENTIFIER); + if (!success) { + console.error('OpenAI rate limit exceeded'); + throw new Error('OpenAI rate limit exceeded'); + } + } + const stream = createStreamableValue(''); render({ diff --git a/src/services/vercel-kv.ts b/src/services/vercel-kv.ts new file mode 100644 index 00000000..f6ef92e8 --- /dev/null +++ b/src/services/vercel-kv.ts @@ -0,0 +1,10 @@ +import { createClient } from '@vercel/kv'; + +export const kv = + process.env.REDIS_REST_API_URL && + process.env.REDIS_REST_API_TOKEN + ? createClient({ + url: process.env.REDIS_REST_API_URL, + token: process.env.REDIS_REST_API_TOKEN, + }) + : undefined; diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 1fb9ec0d..87c2442d 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -20,10 +20,12 @@ import { toastSuccess } from '@/toast'; import { ConfigChecklistStatus } from './config'; import StatusIcon from '@/components/StatusIcon'; import { labelForStorage } from '@/services/storage'; +import { HiSparkles } from 'react-icons/hi'; export default function SiteChecklistClient({ - hasPostgres, - hasStorage, + hasVercelPostgres, + hasVercelKV, + hasStorageProvider, hasVercelBlobStorage, hasCloudflareR2Storage, hasAwsS3Storage, @@ -140,7 +142,7 @@ export default function SiteChecklistClient({ > {renderLink( @@ -152,13 +154,13 @@ export default function SiteChecklistClient({ and connect to project {renderSubStatus( @@ -266,6 +268,38 @@ export default function SiteChecklistClient({ {renderEnvVars(['NEXT_PUBLIC_SITE_DOMAIN'])} + } + optional + > + + Store your OpenAI secret key in order to add experimental support + for AI-generated text descriptions and enable an invisible field + called {'"Semantic Description"'} used to support CMD-K search + {renderEnvVars(['OPENAI_SECRET_KEY'])} + + + {renderLink( + // eslint-disable-next-line max-len + 'https://vercel.com/docs/storage/vercel-kv/quickstart#create-a-kv-database', + 'Create Vercel KV store', + )} + {' '} + and connect to project in order to enable rate limiting + + } @@ -301,18 +335,6 @@ export default function SiteChecklistClient({ collection/display of location-based data {renderEnvVars(['NEXT_PUBLIC_GEO_PRIVACY'])} - - Store your OpenAI secret key in order to add experimental support - for AI-generated text descriptions and enable an invisible field - called {'"Semantic Description"'} used to support CMD-K search - {renderEnvVars(['OPENAI_SECRET_KEY'])} - 0; + +// STORAGE: VERCEL KV +export const HAS_VERCEL_KV = + (process.env.REDIS_REST_API_URL ?? '').length > 0 && + (process.env.REDIS_REST_API_TOKEN ?? '').length > 0; + // STORAGE: VERCEL BLOB export const HAS_VERCEL_BLOB_STORAGE = (process.env.BLOB_READ_WRITE_TOKEN ?? '').length > 0; @@ -102,11 +111,12 @@ export const OG_TEXT_BOTTOM_ALIGNMENT = export const HIGH_DENSITY_GRID = GRID_ASPECT_RATIO <= 1; export const CONFIG_CHECKLIST_STATUS = { - hasPostgres: (process.env.POSTGRES_HOST ?? '').length > 0, + hasVercelPostgres: HAS_VERCEL_POSTGRES, + hasVercelKV: HAS_VERCEL_KV, hasVercelBlobStorage: HAS_VERCEL_BLOB_STORAGE, hasCloudflareR2Storage: HAS_CLOUDFLARE_R2_STORAGE, hasAwsS3Storage: HAS_AWS_S3_STORAGE, - hasStorage: + hasStorageProvider: HAS_VERCEL_BLOB_STORAGE || HAS_CLOUDFLARE_R2_STORAGE || HAS_AWS_S3_STORAGE, @@ -135,7 +145,7 @@ export const CONFIG_CHECKLIST_STATUS = { export type ConfigChecklistStatus = typeof CONFIG_CHECKLIST_STATUS; export const IS_SITE_READY = - CONFIG_CHECKLIST_STATUS.hasPostgres && - CONFIG_CHECKLIST_STATUS.hasStorage && + CONFIG_CHECKLIST_STATUS.hasVercelPostgres && + CONFIG_CHECKLIST_STATUS.hasStorageProvider && CONFIG_CHECKLIST_STATUS.hasAuthSecret && CONFIG_CHECKLIST_STATUS.hasAdminUser; From 195c640efcaba1a30b649fbb9f4f2e230fc727ae Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 19:20:20 -0500 Subject: [PATCH 18/38] Refactor site config checklist --- src/components/Badge.tsx | 3 +++ src/components/Checklist.tsx | 18 ++++++++++++------ src/components/ChecklistRow.tsx | 11 ++--------- src/components/ExperimentalBadge.tsx | 20 ++++++++++++++++++++ src/site/SiteChecklistClient.tsx | 2 +- 5 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 src/components/ExperimentalBadge.tsx diff --git a/src/components/Badge.tsx b/src/components/Badge.tsx index fb4850d4..8a942928 100644 --- a/src/components/Badge.tsx +++ b/src/components/Badge.tsx @@ -7,6 +7,7 @@ export default function Badge({ highContrast, uppercase, interactive, + className, }: { children: React.ReactNode type?: 'large' | 'small' | 'text-only' @@ -14,6 +15,7 @@ export default function Badge({ highContrast?: boolean uppercase?: boolean interactive?: boolean + className?: string }) { const stylesForType = () => { switch (type) { @@ -44,6 +46,7 @@ export default function Badge({ 'leading-none', stylesForType(), uppercase && 'uppercase tracking-wider', + className, )}> {children} diff --git a/src/components/Checklist.tsx b/src/components/Checklist.tsx index cb80dded..62a23bab 100644 --- a/src/components/Checklist.tsx +++ b/src/components/Checklist.tsx @@ -1,30 +1,36 @@ import { ReactNode } from 'react'; import { clsx } from 'clsx/lite'; +import ExperimentalBadge from './ExperimentalBadge'; +import Badge from './Badge'; export default function Checklist({ title, icon, optional, + experimental, children, }: { title: string icon?: ReactNode optional?: boolean + experimental?: boolean children: ReactNode }) { return (
- {icon} -
-
{title}
+ {icon} + + {title} {optional && -
(Optional)
} -
+ Optional} + {experimental && + } +
{title} {experimental && - - Experimental - } + }
{children} diff --git a/src/components/ExperimentalBadge.tsx b/src/components/ExperimentalBadge.tsx new file mode 100644 index 00000000..efeeb59d --- /dev/null +++ b/src/components/ExperimentalBadge.tsx @@ -0,0 +1,20 @@ +import clsx from 'clsx/lite'; +import Badge from './Badge'; + +export default function ExperimentalBadge({ + className, +}: { + className?: string +}) { + return ( + + Experimental + + ); +} diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 87c2442d..31f3c185 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -271,13 +271,13 @@ export default function SiteChecklistClient({ } + experimental optional > Store your OpenAI secret key in order to add experimental support From 1d486634119deab311ad30ee50742194d673dbaf Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 19:23:19 -0500 Subject: [PATCH 19/38] Tweak OpenAI README warning --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 7d84d313..8018f5b9 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,7 @@ Installation ### 7. Add experimental AI text generation -_⚠️ READ BEFORE PROCEEDING_ -- _Usage of this feature will result in fees from OpenAI._ -- _When enabling AI text generation, follow all recommended mitigations in order to avoid unexpected charges and attacks._ -- _Make sure your OpenAI secret key is not prefixed with NEXT_PUBLIC._ +_⚠️ READ BEFORE PROCEEDING: Usage of this feature will result in fees from OpenAI. When enabling AI text generation, follow all recommended mitigations in order to avoid unexpected charges and attacks. Make sure your OpenAI secret key is not prefixed with NEXT_PUBLIC._ 1. Setup OpenAI - If you don't already have one, create an [OpenAI](https://openai.com) account From bbe2fbca8d012bc7d81f9e271b2c8ffc20ec421a Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 19:26:29 -0500 Subject: [PATCH 20/38] Refine OpenAI README warning --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8018f5b9..2349380e 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,9 @@ Installation ### 7. Add experimental AI text generation -_⚠️ READ BEFORE PROCEEDING: Usage of this feature will result in fees from OpenAI. When enabling AI text generation, follow all recommended mitigations in order to avoid unexpected charges and attacks. Make sure your OpenAI secret key is not prefixed with NEXT_PUBLIC._ +_⚠️ READ BEFORE PROCEEDING_ + +> _Usage of this feature will result in fees from OpenAI. When enabling AI text generation, follow all recommended mitigations in order to avoid unexpected charges and attacks. Make sure your OpenAI secret key is not prefixed with NEXT_PUBLIC._ 1. Setup OpenAI - If you don't already have one, create an [OpenAI](https://openai.com) account From 1371a8dcc4846a9b6227e6376a8cc8cf5b3862fc Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 19:43:44 -0500 Subject: [PATCH 21/38] Re-enable standard Vercel KV usage --- src/services/openai.ts | 5 +++-- src/services/vercel-kv.ts | 10 ---------- src/site/SiteChecklistClient.tsx | 2 +- src/site/config.ts | 3 +-- 4 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 src/services/vercel-kv.ts diff --git a/src/services/openai.ts b/src/services/openai.ts index 4ef1661b..7841e134 100644 --- a/src/services/openai.ts +++ b/src/services/openai.ts @@ -2,8 +2,9 @@ import OpenAI from 'openai'; import { createStreamableValue, render } from 'ai/rsc'; -import { kv } from '@/services/vercel-kv'; +import { kv } from '@vercel/kv'; import { Ratelimit } from '@upstash/ratelimit'; +import { HAS_VERCEL_KV } from '@/site/config'; const RATE_LIMIT_IDENTIFIER = 'openai-image-query'; const RATE_LIMIT_MAX_QUERIES_PER_HOUR = 100; @@ -11,7 +12,7 @@ const RATE_LIMIT_MAX_QUERIES_PER_HOUR = 100; const provider = new OpenAI({ apiKey: process.env.OPENAI_SECRET_KEY }); // Allows 100 requests per hour -const ratelimit = kv +const ratelimit = HAS_VERCEL_KV ? new Ratelimit({ redis: kv, limiter: Ratelimit.slidingWindow(RATE_LIMIT_MAX_QUERIES_PER_HOUR, '1h'), diff --git a/src/services/vercel-kv.ts b/src/services/vercel-kv.ts deleted file mode 100644 index f6ef92e8..00000000 --- a/src/services/vercel-kv.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { createClient } from '@vercel/kv'; - -export const kv = - process.env.REDIS_REST_API_URL && - process.env.REDIS_REST_API_TOKEN - ? createClient({ - url: process.env.REDIS_REST_API_URL, - token: process.env.REDIS_REST_API_TOKEN, - }) - : undefined; diff --git a/src/site/SiteChecklistClient.tsx b/src/site/SiteChecklistClient.tsx index 31f3c185..64a36404 100644 --- a/src/site/SiteChecklistClient.tsx +++ b/src/site/SiteChecklistClient.tsx @@ -286,7 +286,7 @@ export default function SiteChecklistClient({ {renderEnvVars(['OPENAI_SECRET_KEY'])} 0 && - (process.env.REDIS_REST_API_TOKEN ?? '').length > 0; + (process.env.KV_URL ?? '').length > 0; // STORAGE: VERCEL BLOB export const HAS_VERCEL_BLOB_STORAGE = From e2e8c8edda343f8bfbb4306d17929ea6c2d769a9 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 23:05:21 -0500 Subject: [PATCH 22/38] Wire up page-level AI streaming --- src/photo/PhotoEditPageClient.tsx | 36 +++-- src/photo/actions.ts | 8 +- src/photo/ai/index.ts | 38 +++-- src/photo/ai/useImageQueries.ts | 55 +++++++ src/photo/ai/useImageQuery.ts | 10 +- src/photo/ai/useTitleCaptionImageQuery.ts | 32 ++++ src/photo/form/PhotoForm.tsx | 173 ++-------------------- src/photo/form/index.ts | 12 +- 8 files changed, 157 insertions(+), 207 deletions(-) create mode 100644 src/photo/ai/useImageQueries.ts create mode 100644 src/photo/ai/useTitleCaptionImageQuery.ts diff --git a/src/photo/PhotoEditPageClient.tsx b/src/photo/PhotoEditPageClient.tsx index a26c054d..ff302454 100644 --- a/src/photo/PhotoEditPageClient.tsx +++ b/src/photo/PhotoEditPageClient.tsx @@ -12,6 +12,9 @@ import IconGrSync from '@/site/IconGrSync'; import { getExifDataAction } from './actions'; import { Tags } from '@/tag'; import { useState } from 'react'; +import useImageQueries from './ai/useImageQueries'; +import { HiSparkles } from 'react-icons/hi'; +import Spinner from '@/components/Spinner'; export default function PhotoEditPageClient({ photo, @@ -37,6 +40,8 @@ export default function PhotoEditPageClient({ seedExifData, ); + const aiContent = useImageQueries(); + return ( - - } +
+ +
+ + } + > + EXIF + +
+
} isLoading={pending} > diff --git a/src/photo/actions.ts b/src/photo/actions.ts index 759e2641..f75855aa 100644 --- a/src/photo/actions.ts +++ b/src/photo/actions.ts @@ -34,7 +34,7 @@ import { extractExifDataFromBlobPath } from './server'; import { TAG_FAVS, isTagFavs } from '@/tag'; import { convertPhotoToPhotoDbInsert } from '.'; import { safelyRunAdminServerAction } from '@/auth'; -import { ImageQuery, streamImageQuery } from './ai'; +import { AiImageQuery, streamAiImageQuery } from './ai'; export async function createPhotoAction(formData: FormData) { return safelyRunAdminServerAction(async () => { @@ -183,10 +183,10 @@ export async function syncCacheAction() { return safelyRunAdminServerAction(revalidateAllKeysAndPaths); } -export async function streamImageQueryAction( +export async function streamAiImageQueryAction( imageBase64: string, - query: ImageQuery, + query: AiImageQuery, ) { return safelyRunAdminServerAction(async () => - streamImageQuery(imageBase64, query)); + streamAiImageQuery(imageBase64, query)); } diff --git a/src/photo/ai/index.ts b/src/photo/ai/index.ts index b8375008..a2bb8669 100644 --- a/src/photo/ai/index.ts +++ b/src/photo/ai/index.ts @@ -1,29 +1,27 @@ +/* eslint-disable max-len */ + import { streamOpenAiImageQuery } from '@/services/openai'; -export type ImageQuery = +export type AiImageQuery = 'title' | 'caption' | + 'title-and-caption' | 'tags' | - 'descriptionSmall' | - 'descriptionMedium' | - 'descriptionLarge' | - 'rich' | + 'description-small' | + 'description' | + 'description-large' | 'semantic'; -export const IMAGE_QUERIES: Record = { - // title: 'Provide a short title for this image', - title: 'Provide a short title for this image in 3 words or less', - caption: 'What is a pithy caption for this image in 8 words or less?', - // eslint-disable-next-line max-len - tags: 'Describe this image three or less comma-separated keywords with no adjective or adverbs', - descriptionSmall: 'Describe this image succinctly', - descriptionMedium: 'Describe this image', - descriptionLarge: 'Describe this image in detail', - // eslint-disable-next-line max-len - rich: 'What is a short title and pithy caption of 8 words or less for this image?', - // eslint-disable-next-line max-len - semantic: 'List up to 5 things in this image without description as a comma-separated list', +export const AI_IMAGE_QUERIES: Record = { + 'title': 'Provide a short title for this image in 3 words or less', + 'caption': 'What is a pithy caption for this image in 8 words or less?', + 'title-and-caption': 'Write a short title and pithy caption of 8 words or less for this image, using the format Title: "title" Caption: "caption"', + 'tags': 'Describe this image three or less comma-separated keywords with no adjective or adverbs', + 'description-small': 'Describe this image succinctly', + 'description': 'Describe this image', + 'description-large': 'Describe this image in detail', + 'semantic': 'List up to 5 things in this image without description as a comma-separated list', }; -export const streamImageQuery = (imageBase64: string, query: ImageQuery) => - streamOpenAiImageQuery(imageBase64, IMAGE_QUERIES[query]); +export const streamAiImageQuery = (imageBase64: string, query: AiImageQuery) => + streamOpenAiImageQuery(imageBase64, AI_IMAGE_QUERIES[query]); diff --git a/src/photo/ai/useImageQueries.ts b/src/photo/ai/useImageQueries.ts new file mode 100644 index 00000000..1fffe33a --- /dev/null +++ b/src/photo/ai/useImageQueries.ts @@ -0,0 +1,55 @@ +import { useCallback, useState } from 'react'; +import useImageQuery from './useImageQuery'; +import useTitleCaptionImageQuery from './useTitleCaptionImageQuery'; + +export type AiContent = ReturnType; + +export default function useImageQueries() { + const [imageData, setImageData] = useState(); + + const isReady = Boolean(imageData); + + const [ + requestTitleCaption, + title, + caption, + isLoadingTitleCaption, + ] = useTitleCaptionImageQuery(imageData); + + const [ + requestTags, + tags, + isLoadingTags, + ] = useImageQuery(imageData, 'tags'); + + const [ + requestSemantic, + semantic, + isLoadingSemantic, + ] = useImageQuery(imageData, 'semantic'); + + const isLoading = isLoadingTitleCaption || isLoadingTags || isLoadingSemantic; + + const request = useCallback(async () => { + if (!isLoading) { + console.log('REQUESTING ALL IMAGE QUERIES'); + requestTitleCaption(); + requestTags(); + requestSemantic(); + } + }, [isLoading, requestTitleCaption, requestTags, requestSemantic]); + + return { + request, + title, + caption, + tags, + semantic, + isReady, + isLoading, + isLoadingTitleCaption, + isLoadingTags, + isLoadingSemantic, + setImageData, + }; +} diff --git a/src/photo/ai/useImageQuery.ts b/src/photo/ai/useImageQuery.ts index 9f0079eb..6eb598ef 100644 --- a/src/photo/ai/useImageQuery.ts +++ b/src/photo/ai/useImageQuery.ts @@ -1,11 +1,11 @@ import { useCallback, useState } from 'react'; -import { streamImageQueryAction } from '../actions'; +import { streamAiImageQueryAction } from '../actions'; import { readStreamableValue } from 'ai/rsc'; -import { ImageQuery } from '.'; +import { AiImageQuery } from '.'; export default function useImageQuery( imageBase64: string | undefined, - query: ImageQuery, + query: AiImageQuery, ) { const [text, setText] = useState(''); const [error, setError] = useState(); @@ -15,12 +15,12 @@ export default function useImageQuery( if (imageBase64) { setIsLoading(true); try { - const textStream = await streamImageQueryAction( + const textStream = await streamAiImageQueryAction( imageBase64, query, ); for await (const text of readStreamableValue(textStream)) { - setText(text ?? ''); + setText((text ?? '').replaceAll('\n', ' ')); } setIsLoading(false); } catch (e) { diff --git a/src/photo/ai/useTitleCaptionImageQuery.ts b/src/photo/ai/useTitleCaptionImageQuery.ts new file mode 100644 index 00000000..c9263fae --- /dev/null +++ b/src/photo/ai/useTitleCaptionImageQuery.ts @@ -0,0 +1,32 @@ +import { useMemo } from 'react'; +import useImageQuery from './useImageQuery'; + +export default function useTitleCaptionImageQuery( + imageBase64: string | undefined, +) { + const [ + request, + text, + isLoading, + error, + ] = useImageQuery(imageBase64, 'title-and-caption'); + + const { title, caption } = useMemo(() => { + const matches = text.includes('Title') + ? text.match(/^[`']*Title: "*(.*?)\.*"* Caption: "*(.*?)\.*"*[`']*$/) + : text.match(/^(.*?): (.*?)$/); + + return { + title: matches?.[1] ?? '', + caption: matches?.[2] ?? '', + }; + }, [text]); + + return [ + request, + title, + caption, + isLoading, + error, + ] as const; +} diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index 0ff11a6e..c2708e51 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -25,8 +25,7 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; -import Spinner from '@/components/Spinner'; -import useImageQuery from '../ai/useImageQuery'; +import { AiContent } from '../ai/useImageQueries'; const THUMBNAIL_SIZE = 300; @@ -35,7 +34,7 @@ export default function PhotoForm({ updatedExifData, type = 'create', uniqueTags, - aiTextGeneration, + aiContent, debugBlur, onTitleChange, onFormStatusChange, @@ -44,7 +43,8 @@ export default function PhotoForm({ updatedExifData?: Partial type?: 'create' | 'edit' uniqueTags?: Tags - aiTextGeneration?: boolean + aiContent?: AiContent + setImageData?: (imageData: string) => void debugBlur?: boolean onTitleChange?: (updatedTitle: string) => void onFormStatusChange?: (pending: boolean) => void @@ -55,8 +55,6 @@ export default function PhotoForm({ useState(getFormErrors(initialPhotoForm)); const [blurError, setBlurError] = useState(); - const [imageData, setImageData] = - useState(); // Update form when EXIF data // is refreshed by parent @@ -122,123 +120,12 @@ export default function PhotoForm({ } }, []); - // const [ - // requestTitle, - // title, - // isLoadingTitle, - // errorTitle, - // ] = useImageQuery(imageData, 'title'); - - // const [ - // requestCaption, - // caption, - // isLoadingCaption, - // errorCaption, - // ] = useImageQuery(imageData, 'caption'); - - const [ - requestTags, - tags, - isLoadingTags, - errorTags, - ] = useImageQuery(imageData, 'tags'); - - const [ - requestRich, - rich, - isLoadingRich, - errorRich, - ] = useImageQuery(imageData, 'rich'); - - // const [ - // requestDescriptionSmall, - // descriptionSmall, - // isLoadingDescriptionSmall, - // errorDescriptionSmall, - // ] = useImageQuery(imageData, 'descriptionSmall'); - - const [ - requestSemantic, - semantic, - isLoadingSemantic, - errorSemantic, - ] = useImageQuery(imageData, 'semantic'); - - const renderAiButton = ( - label: string, - onClick: () => void, - isLoading: boolean, - error?: any, - ) => - ; - return (
- {blurError && + {debugBlur && blurError &&
{blurError}
} -
- {/* {renderAiButton( - 'Title', - requestTitle, - isLoadingTitle, - errorTitle, - )} - {renderAiButton( - 'Caption', - requestCaption, - isLoadingCaption, - errorCaption, - )} - {renderAiButton( - 'Tags', - requestTags, - isLoadingTags, - errorTags, - )} */} - {renderAiButton( - 'Rich', - requestRich, - isLoadingRich, - errorRich, - )} - {renderAiButton( - 'Tags', - requestTags, - isLoadingTags, - errorTags, - )} - {renderAiButton( - 'Semantic', - requestSemantic, - isLoadingSemantic, - errorSemantic, - )} - {/* {renderAiButton( - 'Description', - requestDescriptionSmall, - isLoadingDescriptionSmall, - errorDescriptionSmall, - )} */} -
@@ -271,48 +158,10 @@ export default function PhotoForm({ height={height} />}
- {/*

- ✨ TITLE: {title} {isLoadingTitle && <> - - - - } -

-

- ✨ CAPTION: {caption} {isLoadingCaption && <> - - - - } -

*/} -

- ✨ RICH: {rich} {isLoadingRich && <> - - - - } -

-

- ✨ TAGS: {tags} {isLoadingTags && <> - - - - } -

-

- ✨ SEMANTIC: {semantic} {isLoadingSemantic && <> - - - - } -

- {/*

- ✨ DESCRIPTION: {descriptionSmall} {isLoadingDescriptionSmall && <> - - - - } -

*/} +
Title: {aiContent?.title}
+
Caption: {aiContent?.caption}
+
Tags: {aiContent?.tags}
+
Semantic: {aiContent?.semantic}
blur()} @@ -325,7 +174,7 @@ export default function PhotoForm({ annotation: formatCount(count), annotationAria: formatCountDescriptive(count, 'tagged'), })), - aiTextGeneration, + aiContent !== undefined, ) .map(([key, { label, diff --git a/src/photo/form/index.ts b/src/photo/form/index.ts index 39ec8661..0e4a05cd 100644 --- a/src/photo/form/index.ts +++ b/src/photo/form/index.ts @@ -72,12 +72,6 @@ const FORM_METADATA = ( validateStringMaxLength: STRING_MAX_LENGTH_LONG, shouldHide: ({ title, caption }) => !title && !caption, }, - semanticDescription: { - label: 'semantic description', - capitalize: true, - validateStringMaxLength: STRING_MAX_LENGTH_LONG, - hide: !aiTextGeneration, - }, tags: { label: 'tags', tagOptions, @@ -85,6 +79,12 @@ const FORM_METADATA = ( ? `'${TAG_FAVS}' is a reserved tag` : undefined, }, + semanticDescription: { + label: 'semantic description', + capitalize: true, + validateStringMaxLength: STRING_MAX_LENGTH_LONG, + hide: !aiTextGeneration, + }, id: { label: 'id', readOnly: true, hideIfEmpty: true }, blurData: { label: 'blur data', From 097496a7394309fdbf150e9a068a7bfed0d786ba Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 23:20:42 -0500 Subject: [PATCH 23/38] Integrate ai auto-fill into edit form --- src/components/FieldSetWithStatus.tsx | 4 +-- src/photo/ai/index.ts | 4 +-- src/photo/ai/useImageQueries.ts | 19 ++++++++++++--- src/photo/form/PhotoForm.tsx | 35 +++++++++++++++++++++++---- src/photo/form/index.ts | 2 +- 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/components/FieldSetWithStatus.tsx b/src/components/FieldSetWithStatus.tsx index 19e7ab01..85677d12 100644 --- a/src/components/FieldSetWithStatus.tsx +++ b/src/components/FieldSetWithStatus.tsx @@ -99,7 +99,7 @@ export default function FieldSetWithStatus({ options={tagOptions} onChange={onChange} className={clsx(Boolean(error) && 'error')} - readOnly={readOnly || pending} + readOnly={readOnly || pending || loading} /> : = { 'title': 'Provide a short title for this image in 3 words or less', @@ -20,7 +20,7 @@ export const AI_IMAGE_QUERIES: Record = { 'description-small': 'Describe this image succinctly', 'description': 'Describe this image', 'description-large': 'Describe this image in detail', - 'semantic': 'List up to 5 things in this image without description as a comma-separated list', + 'description-semantic': 'List up to 5 things in this image without description as a comma-separated list', }; export const streamAiImageQuery = (imageBase64: string, query: AiImageQuery) => diff --git a/src/photo/ai/useImageQueries.ts b/src/photo/ai/useImageQueries.ts index 1fffe33a..f0fd504d 100644 --- a/src/photo/ai/useImageQueries.ts +++ b/src/photo/ai/useImageQueries.ts @@ -24,11 +24,21 @@ export default function useImageQueries() { const [ requestSemantic, - semantic, + semanticDescription, isLoadingSemantic, - ] = useImageQuery(imageData, 'semantic'); + ] = useImageQuery(imageData, 'description-semantic'); - const isLoading = isLoadingTitleCaption || isLoadingTags || isLoadingSemantic; + const hasContent = Boolean( + title || + caption || + tags || + semanticDescription + ); + + const isLoading = + isLoadingTitleCaption || + isLoadingTags || + isLoadingSemantic; const request = useCallback(async () => { if (!isLoading) { @@ -44,8 +54,9 @@ export default function useImageQueries() { title, caption, tags, - semantic, + semanticDescription, isReady, + hasContent, isLoading, isLoadingTitleCaption, isLoadingTags, diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index c2708e51..b730f086 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -120,6 +120,33 @@ export default function PhotoForm({ } }, []); + useEffect(() => { + if (aiContent?.hasContent) { + setFormData(data => ({ + ...data, + title: aiContent.title, + caption: aiContent.caption, + tags: aiContent.tags, + semanticDescription: aiContent.semanticDescription, + })); + } + }, [aiContent]); + + const isFieldGeneratingAi = (key: keyof PhotoFormData) => { + switch (key) { + case 'title': + return aiContent?.isLoadingTitleCaption; + case 'caption': + return aiContent?.isLoadingTitleCaption; + case 'tags': + return aiContent?.isLoadingTags; + case 'semanticDescription': + return aiContent?.isLoadingSemantic; + default: + return false; + } + }; + return (
{debugBlur && blurError && @@ -158,10 +185,6 @@ export default function PhotoForm({ height={height} />}
-
Title: {aiContent?.title}
-
Caption: {aiContent?.caption}
-
Tags: {aiContent?.tags}
-
Semantic: {aiContent?.semantic}
blur()} @@ -228,7 +251,9 @@ export default function PhotoForm({ placeholder={loadingMessage && !formData[key] ? loadingMessage : undefined} - loading={loadingMessage && !formData[key] ? true : false} + loading={ + (loadingMessage && !formData[key] ? true : false) || + isFieldGeneratingAi(key)} type={type} />)}
diff --git a/src/photo/form/index.ts b/src/photo/form/index.ts index 0e4a05cd..f602dd4d 100644 --- a/src/photo/form/index.ts +++ b/src/photo/form/index.ts @@ -80,7 +80,7 @@ const FORM_METADATA = ( : undefined, }, semanticDescription: { - label: 'semantic description', + label: 'semantic description (not visible)', capitalize: true, validateStringMaxLength: STRING_MAX_LENGTH_LONG, hide: !aiTextGeneration, From ec828f6977536ef3b8e5ef6744d06cd98a2cfcee Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Wed, 20 Mar 2024 23:34:00 -0500 Subject: [PATCH 24/38] Fix upload page AI incompatibilities --- src/app/admin/uploads/[uploadPath]/page.tsx | 4 ---- src/photo/UploadPageClient.tsx | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/app/admin/uploads/[uploadPath]/page.tsx b/src/app/admin/uploads/[uploadPath]/page.tsx index d075631c..a99a94c8 100644 --- a/src/app/admin/uploads/[uploadPath]/page.tsx +++ b/src/app/admin/uploads/[uploadPath]/page.tsx @@ -3,7 +3,6 @@ import { extractExifDataFromBlobPath } from '@/photo/server'; import { redirect } from 'next/navigation'; import { getUniqueTagsCached } from '@/photo/cache'; import UploadPageClient from '@/photo/UploadPageClient'; -import { AI_TEXT_GENERATION_ENABLED } from '@/site/config'; interface Params { params: { uploadPath: string } @@ -19,14 +18,11 @@ export default async function UploadPage({ params: { uploadPath } }: Params) { const uniqueTags = await getUniqueTagsCached(); - const aiTextGeneration = AI_TEXT_GENERATION_ENABLED; - return ( ); }; diff --git a/src/photo/UploadPageClient.tsx b/src/photo/UploadPageClient.tsx index be755510..b37a4d39 100644 --- a/src/photo/UploadPageClient.tsx +++ b/src/photo/UploadPageClient.tsx @@ -11,12 +11,10 @@ export default function UploadPageClient({ blobId, photoFormExif, uniqueTags, - aiTextGeneration, }: { blobId?: string photoFormExif: Partial uniqueTags: Tags - aiTextGeneration: boolean }) { const [pending, setIsPending] = useState(false); const [updatedTitle, setUpdatedTitle] = useState(''); @@ -33,7 +31,6 @@ export default function UploadPageClient({ From 6fd8ff34e2d1aee96aa696c712195cf9c457f897 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Thu, 21 Mar 2024 08:40:21 -0500 Subject: [PATCH 25/38] Rename AI hooks --- src/photo/PhotoEditPageClient.tsx | 4 ++-- .../{useImageQueries.ts => useAiImageQueries.ts} | 14 +++++++------- .../ai/{useImageQuery.ts => useAiImageQuery.ts} | 2 +- ...mageQuery.ts => useTitleCaptionAiImageQuery.ts} | 6 +++--- src/photo/form/PhotoForm.tsx | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) rename src/photo/ai/{useImageQueries.ts => useAiImageQueries.ts} (73%) rename src/photo/ai/{useImageQuery.ts => useAiImageQuery.ts} (96%) rename src/photo/ai/{useTitleCaptionImageQuery.ts => useTitleCaptionAiImageQuery.ts} (76%) diff --git a/src/photo/PhotoEditPageClient.tsx b/src/photo/PhotoEditPageClient.tsx index ff302454..b814a9cd 100644 --- a/src/photo/PhotoEditPageClient.tsx +++ b/src/photo/PhotoEditPageClient.tsx @@ -12,7 +12,7 @@ import IconGrSync from '@/site/IconGrSync'; import { getExifDataAction } from './actions'; import { Tags } from '@/tag'; import { useState } from 'react'; -import useImageQueries from './ai/useImageQueries'; +import useAiImageQueries from './ai/useAiImageQueries'; import { HiSparkles } from 'react-icons/hi'; import Spinner from '@/components/Spinner'; @@ -40,7 +40,7 @@ export default function PhotoEditPageClient({ seedExifData, ); - const aiContent = useImageQueries(); + const aiContent = useAiImageQueries(); return ( ; +export type AiContent = ReturnType; -export default function useImageQueries() { +export default function useAiImageQueries() { const [imageData, setImageData] = useState(); const isReady = Boolean(imageData); @@ -14,19 +14,19 @@ export default function useImageQueries() { title, caption, isLoadingTitleCaption, - ] = useTitleCaptionImageQuery(imageData); + ] = useTitleCaptionAiImageQuery(imageData); const [ requestTags, tags, isLoadingTags, - ] = useImageQuery(imageData, 'tags'); + ] = useAiImageQuery(imageData, 'tags'); const [ requestSemantic, semanticDescription, isLoadingSemantic, - ] = useImageQuery(imageData, 'description-semantic'); + ] = useAiImageQuery(imageData, 'description-semantic'); const hasContent = Boolean( title || diff --git a/src/photo/ai/useImageQuery.ts b/src/photo/ai/useAiImageQuery.ts similarity index 96% rename from src/photo/ai/useImageQuery.ts rename to src/photo/ai/useAiImageQuery.ts index 6eb598ef..a62803ad 100644 --- a/src/photo/ai/useImageQuery.ts +++ b/src/photo/ai/useAiImageQuery.ts @@ -3,7 +3,7 @@ import { streamAiImageQueryAction } from '../actions'; import { readStreamableValue } from 'ai/rsc'; import { AiImageQuery } from '.'; -export default function useImageQuery( +export default function useAiImageQuery( imageBase64: string | undefined, query: AiImageQuery, ) { diff --git a/src/photo/ai/useTitleCaptionImageQuery.ts b/src/photo/ai/useTitleCaptionAiImageQuery.ts similarity index 76% rename from src/photo/ai/useTitleCaptionImageQuery.ts rename to src/photo/ai/useTitleCaptionAiImageQuery.ts index c9263fae..9dfcd171 100644 --- a/src/photo/ai/useTitleCaptionImageQuery.ts +++ b/src/photo/ai/useTitleCaptionAiImageQuery.ts @@ -1,7 +1,7 @@ import { useMemo } from 'react'; -import useImageQuery from './useImageQuery'; +import useAiImageQuery from './useAiImageQuery'; -export default function useTitleCaptionImageQuery( +export default function useTitleCaptionAiImageQuery( imageBase64: string | undefined, ) { const [ @@ -9,7 +9,7 @@ export default function useTitleCaptionImageQuery( text, isLoading, error, - ] = useImageQuery(imageBase64, 'title-and-caption'); + ] = useAiImageQuery(imageBase64, 'title-and-caption'); const { title, caption } = useMemo(() => { const matches = text.includes('Title') diff --git a/src/photo/form/PhotoForm.tsx b/src/photo/form/PhotoForm.tsx index b730f086..6a24014d 100644 --- a/src/photo/form/PhotoForm.tsx +++ b/src/photo/form/PhotoForm.tsx @@ -25,7 +25,7 @@ import ImageBlurFallback from '@/components/ImageBlurFallback'; import { BLUR_ENABLED } from '@/site/config'; import { Tags, sortTagsObjectWithoutFavs } from '@/tag'; import { formatCount, formatCountDescriptive } from '@/utility/string'; -import { AiContent } from '../ai/useImageQueries'; +import { AiContent } from '../ai/useAiImageQueries'; const THUMBNAIL_SIZE = 300; From 9f087165681a5c85cdee160ac485e11ab8c3e359 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Thu, 21 Mar 2024 09:41:43 -0500 Subject: [PATCH 26/38] Finalize photo editing AI experience --- src/app/admin/photos/[photoId]/edit/page.tsx | 4 +- src/components/FieldSetWithStatus.tsx | 51 ++++++++++++-------- src/photo/PhotoEditPageClient.tsx | 32 ++++++------ src/photo/ai/AiButton.tsx | 28 +++++++++++ src/photo/ai/index.ts | 2 +- src/photo/ai/useAiImageQueries.ts | 12 +++-- src/photo/ai/useAiImageQuery.ts | 4 +- src/photo/ai/useTitleCaptionAiImageQuery.ts | 8 ++- src/photo/form/PhotoForm.tsx | 11 +++-- src/photo/form/index.ts | 15 +++++- src/site/globals.css | 13 +++-- 11 files changed, 124 insertions(+), 56 deletions(-) create mode 100644 src/photo/ai/AiButton.tsx diff --git a/src/app/admin/photos/[photoId]/edit/page.tsx b/src/app/admin/photos/[photoId]/edit/page.tsx index 767ba55b..c9a5c68c 100644 --- a/src/app/admin/photos/[photoId]/edit/page.tsx +++ b/src/app/admin/photos/[photoId]/edit/page.tsx @@ -15,13 +15,13 @@ export default async function PhotoEditPage({ const uniqueTags = await getUniqueTagsCached(); - const aiTextGeneration = AI_TEXT_GENERATION_ENABLED; + const hasAiTextGeneration = AI_TEXT_GENERATION_ENABLED; return ( ); }; diff --git a/src/components/FieldSetWithStatus.tsx b/src/components/FieldSetWithStatus.tsx index 85677d12..d681955d 100644 --- a/src/components/FieldSetWithStatus.tsx +++ b/src/components/FieldSetWithStatus.tsx @@ -101,25 +101,38 @@ export default function FieldSetWithStatus({ className={clsx(Boolean(error) && 'error')} readOnly={readOnly || pending || loading} /> - : onChange?.(type === 'checkbox' - ? e.target.value === 'true' ? 'false' : 'true' - : e.target.value)} - type={type} - autoComplete="off" - autoCapitalize={!capitalize ? 'off' : undefined} - readOnly={readOnly || pending || loading} - className={clsx( - type === 'text' && 'w-full', - Boolean(error) && 'error', - )} - />} + : type === 'textarea' + ?