From 311d7a77afb40e6cdbef6920bfbc1fa7c6572590 Mon Sep 17 00:00:00 2001 From: Sam Becker Date: Mon, 26 May 2025 12:41:47 -0500 Subject: [PATCH] Sitemaps (#260) * Track last modified date for category queries * Remove unused hidden tags queries * Add tags to sitemap * Calculate sitemap validation * Add remaining categories to sitemap.xml * Add photos to sitemap.xml * Finalize sitemap metadata * Guard against missing dates in sitemap.xml --- app/sitemap.ts | 111 ++++++++++++++++++++++++++++++++++++++++++ src/camera/index.ts | 12 ++--- src/category/index.ts | 5 ++ src/film/index.tsx | 12 ++--- src/focal/index.ts | 8 +-- src/lens/index.ts | 12 ++--- src/photo/cache.ts | 8 --- src/photo/db/query.ts | 67 +++++++++++++++---------- src/recipe/index.ts | 8 ++- src/tag/index.ts | 23 ++++++--- 10 files changed, 196 insertions(+), 70 deletions(-) create mode 100644 app/sitemap.ts diff --git a/app/sitemap.ts b/app/sitemap.ts new file mode 100644 index 00000000..1402c661 --- /dev/null +++ b/app/sitemap.ts @@ -0,0 +1,111 @@ +import type { MetadataRoute } from 'next'; +import { getDataForCategoriesCached } from '@/category/cache'; +import { + absolutePathForCamera, + absolutePathForFilm, + absolutePathForFocalLength, + absolutePathForLens, + absolutePathForPhoto, + absolutePathForRecipe, + absolutePathForTag, +} from '@/app/paths'; +import { isTagFavs } from '@/tag'; +import { BASE_URL, GRID_HOMEPAGE_ENABLED } from '@/app/config'; +import { getPhotoIdsAndUpdatedAt } from '@/photo/db/query'; + +// Cache for 24 hours +export const revalidate = 86_400; + +const PRIORITY_HOME = 1; +const PRIORITY_HOME_VIEW = 0.9; +const PRIORITY_CATEGORY_SPECIAL = 0.8; +const PRIORITY_CATEGORY = 0.7; +const PRIORITY_PHOTO = 0.5; + +export default async function sitemap(): Promise { + const [ + { + cameras, + lenses, + tags, + recipes, + films, + focalLengths, + }, + photos, + ] = await Promise.all([ + getDataForCategoriesCached(), + getPhotoIdsAndUpdatedAt(), + ]); + + const lastModifiedSite = [ + ...cameras.map(({ lastModified }) => lastModified), + ...lenses.map(({ lastModified }) => lastModified), + ...tags.map(({ lastModified }) => lastModified), + ...recipes.map(({ lastModified }) => lastModified), + ...films.map(({ lastModified }) => lastModified), + ...focalLengths.map(({ lastModified }) => lastModified), + ...photos.map(({ updatedAt }) => updatedAt), + ] + .filter(Boolean) + .sort((a, b) => b.getTime() - a.getTime())[0]; + + return [ + // Homepage + { + url: BASE_URL!, + priority: PRIORITY_HOME, + lastModified: lastModifiedSite, + }, + // Grid or Feed + { + url: GRID_HOMEPAGE_ENABLED ? `${BASE_URL}/feed` : `${BASE_URL}/grid`, + priority: PRIORITY_HOME_VIEW, + lastModified: lastModifiedSite, + }, + // Cameras + ...cameras.map(({ camera, lastModified }) => ({ + url: absolutePathForCamera(camera), + priority: PRIORITY_CATEGORY, + lastModified, + })), + // Lenses + ...lenses.map(({ lens, lastModified }) => ({ + url: absolutePathForLens(lens), + priority: PRIORITY_CATEGORY, + lastModified, + })), + // Tags + ...tags.map(({ tag, lastModified }) => ({ + url: absolutePathForTag(tag), + priority: isTagFavs(tag) + ? PRIORITY_CATEGORY_SPECIAL + : PRIORITY_CATEGORY, + lastModified, + })), + // Recipes + ...recipes.map(({ recipe, lastModified }) => ({ + url: absolutePathForRecipe(recipe), + priority: PRIORITY_CATEGORY, + lastModified, + })), + // Films + ...films.map(({ film, lastModified }) => ({ + url: absolutePathForFilm(film), + priority: PRIORITY_CATEGORY, + lastModified, + })), + // Focal Lengths + ...focalLengths.map(({ focal, lastModified }) => ({ + url: absolutePathForFocalLength(focal), + priority: PRIORITY_CATEGORY, + lastModified, + })), + // Photos + ...photos.map(({ id, updatedAt }) => ({ + url: absolutePathForPhoto({ photo: id }), + priority: PRIORITY_PHOTO, + lastModified: updatedAt, + })), + ]; +} diff --git a/src/camera/index.ts b/src/camera/index.ts index 403f6615..457ce945 100644 --- a/src/camera/index.ts +++ b/src/camera/index.ts @@ -1,3 +1,4 @@ +import { CategoryQueryMeta } from '@/category'; import type { Photo } from '@/photo'; import { isCameraMakeApple } from '@/platforms/apple'; import { formatSonyModel, isMakeSony } from '@/platforms/sony'; @@ -18,13 +19,12 @@ export interface PhotoCameraProps { params: Promise } -export type CameraWithCount = { +export type CameraWithMeta = { cameraKey: string camera: Camera - count: number -} +} & CategoryQueryMeta; -export type Cameras = CameraWithCount[]; +export type Cameras = CameraWithMeta[]; // Support keys for make-only and model-only camera queries export const createCameraKey = ({ make, model }: Partial) => @@ -42,8 +42,8 @@ export const formatCameraParams = ({ }); export const sortCamerasWithCount = ( - a: CameraWithCount, - b: CameraWithCount, + a: CameraWithMeta, + b: CameraWithMeta, ) => { const aText = formatCameraText(a.camera); const bText = formatCameraText(b.camera); diff --git a/src/category/index.ts b/src/category/index.ts index 4c70238a..cd80986a 100644 --- a/src/category/index.ts +++ b/src/category/index.ts @@ -28,6 +28,11 @@ export const DEFAULT_CATEGORY_KEYS: CategoryKeys = [ 'films', ]; +export interface CategoryQueryMeta { + count: number + lastModified: Date +} + export const getHiddenCategories = (keys: CategoryKeys): CategoryKeys => CATEGORY_KEYS.filter(key => !keys.includes(key)); diff --git a/src/film/index.tsx b/src/film/index.tsx index c8caa283..9771fd78 100644 --- a/src/film/index.tsx +++ b/src/film/index.tsx @@ -20,13 +20,11 @@ import { import { AnnotatedTag } from '@/photo/form'; import PhotoFilmIcon from './PhotoFilmIcon'; import { AppTextState } from '@/i18n/state'; +import { CategoryQueryMeta } from '@/category'; -export type FilmWithCount = { - film: string - count: number -} +export type FilmWithMeta = { film: string } & CategoryQueryMeta -export type Films = FilmWithCount[] +export type Films = FilmWithMeta[] export const labelForFilm = (film: string) => { // Use Fujifilm simulation text when recognized @@ -48,8 +46,8 @@ export const sortFilms = ( ) => films.sort(sortFilmsWithCount); export const sortFilmsWithCount = ( - a: FilmWithCount, - b: FilmWithCount, + a: FilmWithMeta, + b: FilmWithMeta, ) => { const aLabel = labelForFilm(a.film).large; const bLabel = labelForFilm(b.film).large; diff --git a/src/focal/index.ts b/src/focal/index.ts index 8a451c5d..4412c84a 100644 --- a/src/focal/index.ts +++ b/src/focal/index.ts @@ -9,11 +9,11 @@ import { absolutePathForFocalLengthImage, } from '@/app/paths'; import { AppTextState } from '@/i18n/state'; +import { CategoryQueryMeta } from '@/category'; -export type FocalLengths = { - focal: number - count: number -}[] +type FocalLengthWithMeta = { focal: number } & CategoryQueryMeta; + +export type FocalLengths = FocalLengthWithMeta[]; export const getFocalLengthFromString = (focalString?: string) => { const focal = focalString?.match(/^([0-9]+)mm/)?.[1]; diff --git a/src/lens/index.ts b/src/lens/index.ts index d91e0c53..2dbef91e 100644 --- a/src/lens/index.ts +++ b/src/lens/index.ts @@ -3,6 +3,7 @@ import { parameterize } from '@/utility/string'; import { formatAppleLensText, isLensApple } from '../platforms/apple'; import { MISSING_FIELD } from '@/app/paths'; import { formatGoogleLensText, isLensGoogle } from '../platforms/google'; +import { CategoryQueryMeta } from '@/category'; const LENS_PLACEHOLDER: Lens = { make: 'Lens', model: 'Model' }; @@ -21,13 +22,12 @@ export interface LensPhotoProps { params: Promise } -export type LensWithCount = { +export type LensWithMeta = { lensKey: string lens: Lens - count: number -} +} & CategoryQueryMeta; -export type Lenses = LensWithCount[]; +export type Lenses = LensWithMeta[]; export const getLensFromParams = async ( params: Promise, @@ -71,8 +71,8 @@ export const formatLensParams = ({ }); export const sortLensesWithCount = ( - a: LensWithCount, - b: LensWithCount, + a: LensWithMeta, + b: LensWithMeta, ) => { const aText = formatLensText(a.lens); const bText = formatLensText(b.lens); diff --git a/src/photo/cache.ts b/src/photo/cache.ts index aa3ba32b..46fce24c 100644 --- a/src/photo/cache.ts +++ b/src/photo/cache.ts @@ -9,7 +9,6 @@ import { getPhotos, getUniqueCameras, getUniqueTags, - getUniqueTagsHidden, getUniqueFilms, getPhotosNearId, getPhotosMostRecentUpdate, @@ -50,7 +49,6 @@ const KEY_RECIPES = 'recipes'; const KEY_FOCAL_LENGTHS = 'focal-lengths'; // Type keys const KEY_COUNT = 'count'; -const KEY_HIDDEN = 'hidden'; const KEY_DATE_RANGE = 'date-range'; const getPhotosCacheKeyForOption = ( @@ -211,12 +209,6 @@ export const getUniqueTagsCached = [KEY_PHOTOS, KEY_TAGS], ); -export const getUniqueTagsHiddenCached = - unstable_cache( - getUniqueTagsHidden, - [KEY_PHOTOS, KEY_TAGS, KEY_HIDDEN], - ); - export const getUniqueCamerasCached = unstable_cache( getUniqueCameras, diff --git a/src/photo/db/query.ts b/src/photo/db/query.ts index 64f306d0..4c48e661 100644 --- a/src/photo/db/query.ts +++ b/src/photo/db/query.ts @@ -323,73 +323,75 @@ export const getPhotosMostRecentUpdate = async () => export const getUniqueTags = async () => safelyQueryPhotos(() => sql` - SELECT DISTINCT unnest(tags) as tag, COUNT(*) + SELECT DISTINCT unnest(tags) as tag, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE GROUP BY tag ORDER BY tag ASC - `.then(({ rows }): Tags => rows.map(({ tag, count }) => ({ + `.then(({ rows }): Tags => rows.map(({ tag, count, last_modified }) => ({ tag: tag as string, count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueTags'); -export const getUniqueTagsHidden = async () => - safelyQueryPhotos(() => sql` - SELECT DISTINCT unnest(tags) as tag, COUNT(*) - FROM photos - GROUP BY tag - ORDER BY tag ASC - `.then(({ rows }): Tags => rows.map(({ tag, count }) => ({ - tag: tag as string, - count: parseInt(count, 10), - }))) - , 'getUniqueTagsHidden'); - export const getUniqueCameras = async () => safelyQueryPhotos(() => sql` - SELECT DISTINCT make||' '||model as camera, make, model, COUNT(*) + SELECT DISTINCT make||' '||model as camera, make, model, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE AND trim(make) <> '' AND trim(model) <> '' GROUP BY make, model ORDER BY camera ASC - `.then(({ rows }): Cameras => rows.map(({ make, model, count }) => ({ + `.then(({ rows }): Cameras => rows.map(({ + make, model, count, last_modified, + }) => ({ cameraKey: createCameraKey({ make, model }), camera: { make, model }, - count: parseInt(count, 10), + count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueCameras'); export const getUniqueLenses = async () => safelyQueryPhotos(() => sql` SELECT DISTINCT lens_make||' '||lens_model as lens, - lens_make, lens_model, COUNT(*) + lens_make, lens_model, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE AND trim(lens_model) <> '' GROUP BY lens_make, lens_model ORDER BY lens ASC `.then(({ rows }): Lenses => rows - .map(({ lens_make: make, lens_model: model, count }) => ({ + .map(({ lens_make: make, lens_model: model, count, last_modified }) => ({ lensKey: createLensKey({ make, model }), lens: { make, model }, - count: parseInt(count, 10), + count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueLenses'); export const getUniqueRecipes = async () => safelyQueryPhotos(() => sql` - SELECT DISTINCT recipe_title, COUNT(*) + SELECT DISTINCT recipe_title, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE AND recipe_title IS NOT NULL GROUP BY recipe_title ORDER BY recipe_title ASC `.then(({ rows }): Recipes => rows - .map(({ recipe_title, count }) => ({ + .map(({ recipe_title, count, last_modified }) => ({ recipe: recipe_title, count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueRecipes'); @@ -438,29 +440,35 @@ export const updateAllMatchingRecipeTitles = ( export const getUniqueFilms = async () => safelyQueryPhotos(() => sql` - SELECT DISTINCT film, COUNT(*) + SELECT DISTINCT film, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE AND film IS NOT NULL GROUP BY film ORDER BY film ASC `.then(({ rows }): Films => rows - .map(({ film, count }) => ({ + .map(({ film, count, last_modified }) => ({ film, count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueFilms'); export const getUniqueFocalLengths = async () => safelyQueryPhotos(() => sql` - SELECT DISTINCT focal_length, COUNT(*) + SELECT DISTINCT focal_length, + COUNT(*), + MAX(updated_at) as last_modified FROM photos WHERE hidden IS NOT TRUE AND focal_length IS NOT NULL GROUP BY focal_length ORDER BY focal_length ASC `.then(({ rows }): FocalLengths => rows - .map(({ focal_length, count }) => ({ + .map(({ focal_length, count, last_modified }) => ({ focal: parseInt(focal_length, 10), count: parseInt(count, 10), + lastModified: last_modified as Date, }))) , 'getUniqueFocalLengths'); @@ -562,6 +570,13 @@ export const getPublicPhotoIds = async ({ limit }: { limit?: number }) => .then(({ rows }) => rows.map(({ id }) => id as string)) , 'getPublicPhotoIds'); +export const getPhotoIdsAndUpdatedAt = async () => + safelyQueryPhotos(() => + sql`SELECT id, updated_at FROM photos WHERE hidden IS NOT TRUE` + .then(({ rows }) => rows.map(({ id, updated_at }) => + ({ id: id as string, updatedAt: updated_at as Date }))) + , 'getPhotoIdsAndUpdatedAt'); + export const getPhoto = async ( id: string, includeHidden?: boolean, diff --git a/src/recipe/index.ts b/src/recipe/index.ts index 4c61327a..08e5ed9d 100644 --- a/src/recipe/index.ts +++ b/src/recipe/index.ts @@ -9,13 +9,11 @@ import { import { FujifilmRecipe } from '@/platforms/fujifilm/recipe'; import { labelForFilm } from '@/film'; import { AppTextState } from '@/i18n/state'; +import { CategoryQueryMeta } from '@/category'; -export type RecipeWithCount = { - recipe: string - count: number -} +export type RecipeWithMeta = { recipe: string } & CategoryQueryMeta -export type Recipes = RecipeWithCount[] +export type Recipes = RecipeWithMeta[] export interface RecipeProps { title?: string diff --git a/src/tag/index.ts b/src/tag/index.ts index 92ef88b6..05a6b592 100644 --- a/src/tag/index.ts +++ b/src/tag/index.ts @@ -15,17 +15,16 @@ import { formatCount, formatCountDescriptive, } from '@/utility/string'; -import { sortCategoryByCount } from '@/category'; +import { CategoryQueryMeta, sortCategoryByCount } from '@/category'; import { AppTextState } from '@/i18n/state'; // Reserved tags export const TAG_FAVS = 'favs'; export const TAG_HIDDEN = 'hidden'; -export type Tags = { - tag: string - count: number -}[] +type TagWithMeta = { tag: string } & CategoryQueryMeta; + +export type Tags = TagWithMeta[] export const formatTag = (tag?: string) => capitalizeWords(tag?.replaceAll('-', ' ')); @@ -138,11 +137,19 @@ export const isPathFavs = (pathname?: string) => export const isTagHidden = (tag: string) => tag.toLowerCase() === TAG_HIDDEN; -export const addHiddenToTags = (tags: Tags, photosCountHidden = 0) => - photosCountHidden > 0 +export const addHiddenToTags = ( + tags: Tags, + countHidden = 0, + lastModifiedHidden = new Date(), +) => + countHidden > 0 ? tags .filter(({ tag }) => tag === TAG_FAVS) - .concat({ tag: TAG_HIDDEN, count: photosCountHidden }) + .concat({ + tag: TAG_HIDDEN, + count: countHidden, + lastModified: lastModifiedHidden, + }) .concat(tags .filter(({ tag }) => tag !== TAG_FAVS) .sort(sortCategoryByCount),