Sitemaps (#260)

* Track last modified date for category queries

* Remove unused hidden tags queries

* Add tags to sitemap

* Calculate sitemap validation

* Add remaining categories to sitemap.xml

* Add photos to sitemap.xml

* Finalize sitemap metadata

* Guard against missing dates in sitemap.xml

app/sitemap.ts

@@ -0,0 +1,111 @@
+import type { MetadataRoute } from 'next';
+import { getDataForCategoriesCached } from '@/category/cache';
+import {
+  absolutePathForCamera,
+  absolutePathForFilm,
+  absolutePathForFocalLength,
+  absolutePathForLens,
+  absolutePathForPhoto,
+  absolutePathForRecipe,
+  absolutePathForTag,
+} from '@/app/paths';
+import { isTagFavs } from '@/tag';
+import { BASE_URL, GRID_HOMEPAGE_ENABLED } from '@/app/config';
+import { getPhotoIdsAndUpdatedAt } from '@/photo/db/query';
+
+// Cache for 24 hours
+export const revalidate = 86_400;
+
+const PRIORITY_HOME = 1;
+const PRIORITY_HOME_VIEW = 0.9;
+const PRIORITY_CATEGORY_SPECIAL = 0.8;
+const PRIORITY_CATEGORY = 0.7;
+const PRIORITY_PHOTO = 0.5;
+ 
+export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+  const [
+    {
+      cameras,
+      lenses,
+      tags,
+      recipes,
+      films,
+      focalLengths,
+    },
+    photos,
+  ] = await Promise.all([
+    getDataForCategoriesCached(),
+    getPhotoIdsAndUpdatedAt(),
+  ]);
+
+  const lastModifiedSite = [
+    ...cameras.map(({ lastModified }) => lastModified),
+    ...lenses.map(({ lastModified }) => lastModified),
+    ...tags.map(({ lastModified }) => lastModified),
+    ...recipes.map(({ lastModified }) => lastModified),
+    ...films.map(({ lastModified }) => lastModified),
+    ...focalLengths.map(({ lastModified }) => lastModified),
+    ...photos.map(({ updatedAt }) => updatedAt),
+  ]
+    .filter(Boolean)
+    .sort((a, b) => b.getTime() - a.getTime())[0];
+
+  return [
+    // Homepage
+    {
+      url: BASE_URL!,
+      priority: PRIORITY_HOME,
+      lastModified: lastModifiedSite,
+    },
+    // Grid or Feed
+    {
+      url: GRID_HOMEPAGE_ENABLED ? `${BASE_URL}/feed` : `${BASE_URL}/grid`,
+      priority: PRIORITY_HOME_VIEW,
+      lastModified: lastModifiedSite,
+    },
+    // Cameras
+    ...cameras.map(({ camera, lastModified }) => ({
+      url: absolutePathForCamera(camera),
+      priority: PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Lenses
+    ...lenses.map(({ lens, lastModified }) => ({
+      url: absolutePathForLens(lens),
+      priority: PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Tags
+    ...tags.map(({ tag, lastModified }) => ({
+      url: absolutePathForTag(tag),
+      priority: isTagFavs(tag)
+        ? PRIORITY_CATEGORY_SPECIAL
+        : PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Recipes
+    ...recipes.map(({ recipe, lastModified }) => ({
+      url: absolutePathForRecipe(recipe),
+      priority: PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Films
+    ...films.map(({ film, lastModified }) => ({
+      url: absolutePathForFilm(film),
+      priority: PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Focal Lengths
+    ...focalLengths.map(({ focal, lastModified }) => ({
+      url: absolutePathForFocalLength(focal),
+      priority: PRIORITY_CATEGORY,
+      lastModified,
+    })),
+    // Photos
+    ...photos.map(({ id, updatedAt }) => ({
+      url: absolutePathForPhoto({ photo: id }),
+      priority: PRIORITY_PHOTO,
+      lastModified: updatedAt,
+    })),
+  ];
+}

src/camera/index.ts

@@ -1,3 +1,4 @@
+import { CategoryQueryMeta } from '@/category';
 import type { Photo } from '@/photo';
 import { isCameraMakeApple } from '@/platforms/apple';
 import { formatSonyModel, isMakeSony } from '@/platforms/sony';
@@ -18,13 +19,12 @@ export interface PhotoCameraProps {
   params: Promise<Camera & { photoId: string }>
 }
 
-export type CameraWithCount = {
+export type CameraWithMeta = {
   cameraKey: string
   camera: Camera
-  count: number
+} & CategoryQueryMeta;
-}
 
-export type Cameras = CameraWithCount[];
+export type Cameras = CameraWithMeta[];
 
 // Support keys for make-only and model-only camera queries
 export const createCameraKey = ({ make, model }: Partial<Camera>) =>
@@ -42,8 +42,8 @@ export const formatCameraParams = ({
 });
 
 export const sortCamerasWithCount = (
-  a: CameraWithCount,
+  a: CameraWithMeta,
-  b: CameraWithCount,
+  b: CameraWithMeta,
 ) => {
   const aText = formatCameraText(a.camera);
   const bText = formatCameraText(b.camera);

src/category/index.ts

@@ -28,6 +28,11 @@ export const DEFAULT_CATEGORY_KEYS: CategoryKeys = [
   'films',
 ];
 
+export interface CategoryQueryMeta {
+  count: number
+  lastModified: Date
+}
+
 export const getHiddenCategories = (keys: CategoryKeys): CategoryKeys =>
   CATEGORY_KEYS.filter(key => !keys.includes(key));
 

src/film/index.tsx

@@ -20,13 +20,11 @@ import {
 import { AnnotatedTag } from '@/photo/form';
 import PhotoFilmIcon from './PhotoFilmIcon';
 import { AppTextState } from '@/i18n/state';
+import { CategoryQueryMeta } from '@/category';
 
-export type FilmWithCount = {
+export type FilmWithMeta = { film: string } & CategoryQueryMeta
-  film: string
-  count: number
-}
 
-export type Films = FilmWithCount[]
+export type Films = FilmWithMeta[]
 
 export const labelForFilm = (film: string) => {
   // Use Fujifilm simulation text when recognized
@@ -48,8 +46,8 @@ export const sortFilms = (
 ) => films.sort(sortFilmsWithCount);
 
 export const sortFilmsWithCount = (
-  a: FilmWithCount,
+  a: FilmWithMeta,
-  b: FilmWithCount,
+  b: FilmWithMeta,
 ) => {
   const aLabel = labelForFilm(a.film).large;
   const bLabel = labelForFilm(b.film).large;

src/focal/index.ts

@@ -9,11 +9,11 @@ import {
   absolutePathForFocalLengthImage,
 } from '@/app/paths';
 import { AppTextState } from '@/i18n/state';
+import { CategoryQueryMeta } from '@/category';
 
-export type FocalLengths = {
+type FocalLengthWithMeta = { focal: number } & CategoryQueryMeta;
-  focal: number
+
-  count: number
+export type FocalLengths = FocalLengthWithMeta[];
-}[]
 
 export const getFocalLengthFromString = (focalString?: string) => {
   const focal = focalString?.match(/^([0-9]+)mm/)?.[1];

src/lens/index.ts

@@ -3,6 +3,7 @@ import { parameterize } from '@/utility/string';
 import { formatAppleLensText, isLensApple } from '../platforms/apple';
 import { MISSING_FIELD } from '@/app/paths';
 import { formatGoogleLensText, isLensGoogle } from '../platforms/google';
+import { CategoryQueryMeta } from '@/category';
 
 const LENS_PLACEHOLDER: Lens = { make: 'Lens', model: 'Model' };
 
@@ -21,13 +22,12 @@ export interface LensPhotoProps {
   params: Promise<LensWithPhotoId>
 }
 
-export type LensWithCount = {
+export type LensWithMeta = {
   lensKey: string
   lens: Lens
-  count: number
+} & CategoryQueryMeta;
-}
 
-export type Lenses = LensWithCount[];
+export type Lenses = LensWithMeta[];
 
 export const getLensFromParams = async (
   params: Promise<Lens>,
@@ -71,8 +71,8 @@ export const formatLensParams = ({
 });
 
 export const sortLensesWithCount = (
-  a: LensWithCount,
+  a: LensWithMeta,
-  b: LensWithCount,
+  b: LensWithMeta,
 ) => {
   const aText = formatLensText(a.lens);
   const bText = formatLensText(b.lens);

src/photo/cache.ts

@@ -9,7 +9,6 @@ import {
   getPhotos,
   getUniqueCameras,
   getUniqueTags,
-  getUniqueTagsHidden,
   getUniqueFilms,
   getPhotosNearId,
   getPhotosMostRecentUpdate,
@@ -50,7 +49,6 @@ const KEY_RECIPES           = 'recipes';
 const KEY_FOCAL_LENGTHS     = 'focal-lengths';
 // Type keys
 const KEY_COUNT             = 'count';
-const KEY_HIDDEN            = 'hidden';
 const KEY_DATE_RANGE        = 'date-range';
 
 const getPhotosCacheKeyForOption = (
@@ -211,12 +209,6 @@ export const getUniqueTagsCached =
     [KEY_PHOTOS, KEY_TAGS],
   );
 
-export const getUniqueTagsHiddenCached =
-  unstable_cache(
-    getUniqueTagsHidden,
-    [KEY_PHOTOS, KEY_TAGS, KEY_HIDDEN],
-  );
-
 export const getUniqueCamerasCached =
   unstable_cache(
     getUniqueCameras,

src/photo/db/query.ts

@@ -323,73 +323,75 @@ export const getPhotosMostRecentUpdate = async () =>
 
 export const getUniqueTags = async () =>
   safelyQueryPhotos(() => sql`
-    SELECT DISTINCT unnest(tags) as tag, COUNT(*)
+    SELECT DISTINCT unnest(tags) as tag,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE
     GROUP BY tag
     ORDER BY tag ASC
-  `.then(({ rows }): Tags => rows.map(({ tag, count }) => ({
+  `.then(({ rows }): Tags => rows.map(({ tag, count, last_modified }) => ({
       tag: tag as string,
       count: parseInt(count, 10),
+      lastModified: last_modified as Date,
     })))
   , 'getUniqueTags');
 
-export const getUniqueTagsHidden = async () =>
-  safelyQueryPhotos(() => sql`
-    SELECT DISTINCT unnest(tags) as tag, COUNT(*)
-    FROM photos
-    GROUP BY tag
-    ORDER BY tag ASC
-  `.then(({ rows }): Tags => rows.map(({ tag, count }) => ({
-      tag: tag as string,
-      count: parseInt(count, 10),
-    })))
-  , 'getUniqueTagsHidden');
-
 export const getUniqueCameras = async () =>
   safelyQueryPhotos(() => sql`
-    SELECT DISTINCT make||' '||model as camera, make, model, COUNT(*)
+    SELECT DISTINCT make||' '||model as camera, make, model,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE
     AND trim(make) <> ''
     AND trim(model) <> ''
     GROUP BY make, model
     ORDER BY camera ASC
-  `.then(({ rows }): Cameras => rows.map(({ make, model, count }) => ({
+  `.then(({ rows }): Cameras => rows.map(({
+      make, model, count, last_modified,
+    }) => ({
       cameraKey: createCameraKey({ make, model }),
       camera: { make, model },
-      count: parseInt(count, 10),
+      count: parseInt(count, 10), 
+      lastModified: last_modified as Date,
     })))
   , 'getUniqueCameras');
 
 export const getUniqueLenses = async () =>
   safelyQueryPhotos(() => sql`
     SELECT DISTINCT lens_make||' '||lens_model as lens,
-    lens_make, lens_model, COUNT(*)
+      lens_make, lens_model,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE
     AND trim(lens_model) <> ''
     GROUP BY lens_make, lens_model
     ORDER BY lens ASC
   `.then(({ rows }): Lenses => rows
-      .map(({ lens_make: make, lens_model: model, count }) => ({
+      .map(({ lens_make: make, lens_model: model, count, last_modified }) => ({
         lensKey: createLensKey({ make, model }),
         lens: { make, model },
-        count: parseInt(count, 10),
+        count: parseInt(count, 10), 
+        lastModified: last_modified as Date,
       })))
   , 'getUniqueLenses');
 
 export const getUniqueRecipes = async () =>
   safelyQueryPhotos(() => sql`
-    SELECT DISTINCT recipe_title, COUNT(*)
+    SELECT DISTINCT recipe_title,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE AND recipe_title IS NOT NULL
     GROUP BY recipe_title
     ORDER BY recipe_title ASC
   `.then(({ rows }): Recipes => rows
-      .map(({ recipe_title, count }) => ({
+      .map(({ recipe_title, count, last_modified }) => ({
         recipe: recipe_title,
         count: parseInt(count, 10),
+        lastModified: last_modified as Date,
       })))
   , 'getUniqueRecipes');
 
@@ -438,29 +440,35 @@ export const updateAllMatchingRecipeTitles = (
 
 export const getUniqueFilms = async () =>
   safelyQueryPhotos(() => sql`
-    SELECT DISTINCT film, COUNT(*)
+    SELECT DISTINCT film,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE AND film IS NOT NULL
     GROUP BY film
     ORDER BY film ASC
   `.then(({ rows }): Films => rows
-      .map(({ film, count }) => ({
+      .map(({ film, count, last_modified }) => ({
         film,
         count: parseInt(count, 10),
+        lastModified: last_modified as Date,
       })))
   , 'getUniqueFilms');
 
 export const getUniqueFocalLengths = async () =>
   safelyQueryPhotos(() => sql`
-    SELECT DISTINCT focal_length, COUNT(*)
+    SELECT DISTINCT focal_length,
+      COUNT(*),
+      MAX(updated_at) as last_modified
     FROM photos
     WHERE hidden IS NOT TRUE AND focal_length IS NOT NULL
     GROUP BY focal_length
     ORDER BY focal_length ASC
   `.then(({ rows }): FocalLengths => rows
-      .map(({ focal_length, count }) => ({
+      .map(({ focal_length, count, last_modified }) => ({
         focal: parseInt(focal_length, 10),
         count: parseInt(count, 10),
+        lastModified: last_modified as Date,
       })))
   , 'getUniqueFocalLengths');
 
@@ -562,6 +570,13 @@ export const getPublicPhotoIds = async ({ limit }: { limit?: number }) =>
     .then(({ rows }) => rows.map(({ id }) => id as string))
   , 'getPublicPhotoIds');
 
+export const getPhotoIdsAndUpdatedAt = async () =>
+  safelyQueryPhotos(() =>
+    sql`SELECT id, updated_at FROM photos WHERE hidden IS NOT TRUE`
+      .then(({ rows }) => rows.map(({ id, updated_at }) =>
+        ({ id: id as string, updatedAt: updated_at as Date })))
+  , 'getPhotoIdsAndUpdatedAt');
+
 export const getPhoto = async (
   id: string,
   includeHidden?: boolean,

src/recipe/index.ts

@@ -9,13 +9,11 @@ import {
 import { FujifilmRecipe } from '@/platforms/fujifilm/recipe';
 import { labelForFilm } from '@/film';
 import { AppTextState } from '@/i18n/state';
+import { CategoryQueryMeta } from '@/category';
 
-export type RecipeWithCount = {
+export type RecipeWithMeta = { recipe: string } & CategoryQueryMeta
-  recipe: string
-  count: number
-}
 
-export type Recipes = RecipeWithCount[]
+export type Recipes = RecipeWithMeta[]
 
 export interface RecipeProps {
   title?: string

src/tag/index.ts

@@ -15,17 +15,16 @@ import {
   formatCount,
   formatCountDescriptive,
 } from '@/utility/string';
-import { sortCategoryByCount } from '@/category';
+import { CategoryQueryMeta, sortCategoryByCount } from '@/category';
 import { AppTextState } from '@/i18n/state';
 
 // Reserved tags
 export const TAG_FAVS   = 'favs';
 export const TAG_HIDDEN = 'hidden';
 
-export type Tags = {
+type TagWithMeta = { tag: string } & CategoryQueryMeta;
-  tag: string
+
-  count: number
+export type Tags = TagWithMeta[]
-}[]
 
 export const formatTag = (tag?: string) =>
   capitalizeWords(tag?.replaceAll('-', ' '));
@@ -138,11 +137,19 @@ export const isPathFavs = (pathname?: string) =>
 
 export const isTagHidden = (tag: string) => tag.toLowerCase() === TAG_HIDDEN;
 
-export const addHiddenToTags = (tags: Tags, photosCountHidden = 0) =>
+export const addHiddenToTags = (
-  photosCountHidden > 0
+  tags: Tags,
+  countHidden = 0,
+  lastModifiedHidden = new Date(),
+) =>
+  countHidden > 0
     ? tags
       .filter(({ tag }) => tag === TAG_FAVS)
-      .concat({ tag: TAG_HIDDEN, count: photosCountHidden })
+      .concat({
+        tag: TAG_HIDDEN,
+        count: countHidden,
+        lastModified: lastModifiedHidden,
+      })
       .concat(tags
         .filter(({ tag }) => tag !== TAG_FAVS)
         .sort(sortCategoryByCount),