From 7d45b693053b4e76357e59ff2d0f637892b35e2a Mon Sep 17 00:00:00 2001 From: Thom Lamb Date: Sat, 29 Aug 2026 10:45:01 -0500 Subject: [PATCH 1/4] build(uploads): add sharp for image re-encoding (#226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified where it actually has to run rather than only here: the production image builds and `require('sharp')` succeeds inside it on Node v20.20.2, linux/x64, with libvips 8.18.6 and `withExif` available, needing no build toolchain. The architecture question is already settled by this same node:20-bookworm-slim base running in production today, and sharp ships glibc prebuilds for both linux-x64 and linux-arm64, so it adds no constraint that deployment did not already satisfy. Installing it locally found a trap worth recording. sharp requires Node >=20.9.0 and its platform binary is an *optional* dependency, so npm skips it when the engine check fails and still reports success. Installed under this machine's default 18.16.1 the result is a node_modules that looks complete and throws `Could not load the "sharp" module using the win32-x64 runtime` at require time — which reads as a broken package rather than as a wrong Node version. The fix is `npm install --include=optional sharp` under Node 20+, and the prevention is using start-local.ps1 or run-tests.ps1, which switch first. `engines` is now declared so npm at least warns, and the README's existing Node 20 section says what the failure looks like, since the error message names a runtime rather than a version and points nowhere useful. The lockfile carries every platform variant including linux-x64 and linux-arm64, so a build on another platform resolves correctly. The Dockerfile does not copy the lockfile at all and installs fresh, so this matters for contributors rather than for the image. Ref #226 --- README.md | 2 + backend/package-lock.json | 559 +++++++++++++++++++++++++++++++++++++- backend/package.json | 6 +- 3 files changed, 565 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a22d78e..8a766fc 100755 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ Both scripts run `nvm use latest` first and verify the result is Node 20 or newe The Node 20 floor is not arbitrary: `node-pg-migrate` pulls in an `lru-cache` that calls `diagnostics_channel.tracingChannel()`, which does not exist before Node 19.9. On Node 18 migrations die inside minified library code with `(0 , U.tracingChannel) is not a function`, which says nothing about versions. +Since #226 the floor also applies to `npm install`, and it fails in a nastier way. `sharp` declares `>=20.9.0`, and its platform binary — the part that does the actual work — is an **optional** dependency. npm skips an optional dependency whose engine check fails, and reports success. So installing on the machine's default 18.16.1 produces a `node_modules` that looks complete and then throws `Could not load the "sharp" module using the win32-x64 runtime` at require time, which reads as a broken package rather than as a wrong Node version. `backend/package.json` now declares `engines` so npm at least warns; installing through `start-local.ps1` or `run-tests.ps1` avoids it entirely, because both switch first. If you hit it, `npm install --include=optional sharp` under Node 20+ repairs it. + `run-tests.ps1` brings up whatever a suite needs: the integration suite gets its own throwaway Postgres, started and stopped around the run (`-KeepTestDb` leaves it up, `-TestDbPort` moves it if the default is taken or Hyper-V has reserved it). The e2e suite needs the app stack, so start it with `start-local.ps1` first — the script checks and says so rather than letting every spec fail on a refused connection. `-Filter` passes through to the runner to select tests by file or name. ### 1. Start a local Postgres instance diff --git a/backend/package-lock.json b/backend/package-lock.json index 4653eda..e45908e 100755 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -18,7 +18,8 @@ "node-cron": "^3.0.3", "node-pg-migrate": "^7.6.1", "nodemailer": "^6.9.14", - "pg": "^8.12.0" + "pg": "^8.12.0", + "sharp": "^0.35.4" }, "devDependencies": { "@eslint/js": "^9.39.5", @@ -589,6 +590,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -1334,6 +1344,479 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", + "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", + "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", + "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", + "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", + "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", + "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", + "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", + "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", + "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", + "cpu": [ + "riscv64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", + "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", + "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", + "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", + "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", + "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", + "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", + "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", + "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", + "cpu": [ + "riscv64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", + "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", + "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", + "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", + "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", + "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", + "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", + "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", + "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", + "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", @@ -3339,6 +3822,14 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -6863,6 +7354,66 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", + "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.4", + "@img/sharp-darwin-x64": "0.35.4", + "@img/sharp-freebsd-wasm32": "0.35.4", + "@img/sharp-libvips-darwin-arm64": "1.3.3", + "@img/sharp-libvips-darwin-x64": "1.3.3", + "@img/sharp-libvips-linux-arm": "1.3.3", + "@img/sharp-libvips-linux-arm64": "1.3.3", + "@img/sharp-libvips-linux-ppc64": "1.3.3", + "@img/sharp-libvips-linux-riscv64": "1.3.3", + "@img/sharp-libvips-linux-s390x": "1.3.3", + "@img/sharp-libvips-linux-x64": "1.3.3", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", + "@img/sharp-libvips-linuxmusl-x64": "1.3.3", + "@img/sharp-linux-arm": "0.35.4", + "@img/sharp-linux-arm64": "0.35.4", + "@img/sharp-linux-ppc64": "0.35.4", + "@img/sharp-linux-riscv64": "0.35.4", + "@img/sharp-linux-s390x": "0.35.4", + "@img/sharp-linux-x64": "0.35.4", + "@img/sharp-linuxmusl-arm64": "0.35.4", + "@img/sharp-linuxmusl-x64": "0.35.4", + "@img/sharp-webcontainers-wasm32": "0.35.4", + "@img/sharp-win32-arm64": "0.35.4", + "@img/sharp-win32-ia32": "0.35.4", + "@img/sharp-win32-x64": "0.35.4" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -7429,6 +7980,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "optional": true + }, "node_modules/tsx": { "version": "4.23.12", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", diff --git a/backend/package.json b/backend/package.json index 924433d..4d250b9 100755 --- a/backend/package.json +++ b/backend/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "private": true, "main": "dist/server.js", + "engines": { + "node": ">=20.9.0" + }, "scripts": { "build": "tsc", "lint": "eslint src scripts", @@ -33,7 +36,8 @@ "node-cron": "^3.0.3", "node-pg-migrate": "^7.6.1", "nodemailer": "^6.9.14", - "pg": "^8.12.0" + "pg": "^8.12.0", + "sharp": "^0.35.4" }, "devDependencies": { "@eslint/js": "^9.39.5", From e85be0f970ea50c1e1fa91f2b8579a39022803de Mon Sep 17 00:00:00 2001 From: Thom Lamb Date: Sat, 29 Aug 2026 10:55:33 -0500 Subject: [PATCH 2/4] feat(uploads): re-encode images to strip metadata and bound dimensions (#226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-encoding rather than deleting tags. Deleting requires knowing every tag that could carry something sensitive, across formats and camera makers, indefinitely; rebuilding the file from decoded pixels leaves nothing that could have been missed. The same reasoning that makes uploadTypes.ts an allowlist rather than a denylist. `needsProcessing` is pure and separately tested because it is the whole of the backfill's idempotency argument: a file with no EXIF already inside the bounds is already in its final state, so a second run skips it instead of putting it through another lossy pass. Being wrong there degrades every image a little more on every run. Anything sharp cannot describe is processed rather than skipped, since a file we understand least is not one to assume is safe. Verified end to end on a real image before wiring anything up: 3000x2000 with EXIF present became 2000x1333 with EXIF absent, and no temporary file was left behind. Corrects something this README claimed an hour ago. Installing under a Node below 20.9.0 does produce a broken sharp, because npm skips the optional platform binary when the engine check fails and still reports success. But once that binary is present sharp loads and runs fine on 18.16.1 — `engines` is enforced at install time, not at require time. The README said the runtime was blocked, which would have sent someone switching Node versions to fix a problem that only the install created. Ref #226 --- README.md | 4 +- backend/src/imageProcessing.ts | 108 +++++++++++++++++++++ backend/tests/unit/imageProcessing.test.ts | 31 ++++++ 3 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 backend/src/imageProcessing.ts create mode 100644 backend/tests/unit/imageProcessing.test.ts diff --git a/README.md b/README.md index 8a766fc..f1b308b 100755 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ Both scripts run `nvm use latest` first and verify the result is Node 20 or newe The Node 20 floor is not arbitrary: `node-pg-migrate` pulls in an `lru-cache` that calls `diagnostics_channel.tracingChannel()`, which does not exist before Node 19.9. On Node 18 migrations die inside minified library code with `(0 , U.tracingChannel) is not a function`, which says nothing about versions. -Since #226 the floor also applies to `npm install`, and it fails in a nastier way. `sharp` declares `>=20.9.0`, and its platform binary — the part that does the actual work — is an **optional** dependency. npm skips an optional dependency whose engine check fails, and reports success. So installing on the machine's default 18.16.1 produces a `node_modules` that looks complete and then throws `Could not load the "sharp" module using the win32-x64 runtime` at require time, which reads as a broken package rather than as a wrong Node version. `backend/package.json` now declares `engines` so npm at least warns; installing through `start-local.ps1` or `run-tests.ps1` avoids it entirely, because both switch first. If you hit it, `npm install --include=optional sharp` under Node 20+ repairs it. +Since #226 there is a second Node floor, and it bites at **install** time rather than at run time. `sharp` declares `>=20.9.0`, and the platform binary that does its actual work is an **optional** dependency. npm silently skips an optional dependency whose engine check fails and still reports success — so `npm install` on the machine's default 18.16.1 produces a `node_modules` that looks complete and then throws `Could not load the "sharp" module using the win32-x64 runtime` at require time. That message names a runtime rather than a version and sends you looking in the wrong place. + +Once the binary is installed, sharp loads and runs perfectly well on 18.16.1 — `engines` is advisory at run time. So this is purely about how the install was done, not about which Node runs the tests. Install through `start-local.ps1` or `run-tests.ps1`, which switch to Node 20+ first; if you have already hit it, `npm install --include=optional sharp` under Node 20+ repairs it in place. `run-tests.ps1` brings up whatever a suite needs: the integration suite gets its own throwaway Postgres, started and stopped around the run (`-KeepTestDb` leaves it up, `-TestDbPort` moves it if the default is taken or Hyper-V has reserved it). The e2e suite needs the app stack, so start it with `start-local.ps1` first — the script checks and says so rather than letting every spec fail on a refused connection. `-Filter` passes through to the runner to select tests by file or name. diff --git a/backend/src/imageProcessing.ts b/backend/src/imageProcessing.ts new file mode 100644 index 0000000..0314fb2 --- /dev/null +++ b/backend/src/imageProcessing.ts @@ -0,0 +1,108 @@ +import sharp, { type Sharp } from 'sharp'; +import { promises as fs } from 'fs'; + +/** + * Rebuilding an uploaded image so it carries nothing but the picture. + * + * A file arrives as the camera wrote it, and a camera writes EXIF — which + * routinely includes the coordinates the photo was taken at. Those files are + * served publicly from /uploads/, so an unmodified product photo publishes the + * location it was taken. Nobody sending in a photograph of a vase expects that. + * + * The fix is to re-encode rather than to delete tags. Deleting requires knowing + * every tag that could carry something sensitive, across formats and camera + * makers, forever. Re-encoding builds a new file from the decoded pixels, so + * there is nothing left that could have been missed — the same reasoning that + * makes uploadTypes.ts an allowlist rather than a denylist. + * + * Format is deliberately preserved. Converting to WebP would compress better, + * but it changes stored extensions, and therefore item_images.image_path, and + * therefore turns the backfill into a rename with a window where rows point at + * files that no longer exist. See #226. + */ + +/** Comfortably larger than anything the storefront renders. */ +export const MAX_DIMENSION = 2000; + +/** Where further reduction starts to show on a photograph. */ +export const QUALITY = 82; + +interface ImageFacts { + width?: number; + height?: number; + exif?: unknown; +} + +/** + * Whether a file still needs rebuilding. + * + * Pure, and the backfill's entire idempotency argument: a file with no EXIF + * that is already within bounds is already in its final state, so a re-run + * skips it rather than putting it through a second lossy pass. Anything + * unreadable is processed rather than skipped — a file we cannot describe is + * not one to assume is safe. + */ +export function needsProcessing(meta: ImageFacts): boolean { + if (meta.exif !== undefined && meta.exif !== null) return true; + if (meta.width === undefined || meta.height === undefined) return true; + return meta.width > MAX_DIMENSION || meta.height > MAX_DIMENSION; +} + +function encoderFor(instance: Sharp, mimetype: string): Sharp { + switch (mimetype) { + case 'image/jpeg': + return instance.jpeg({ quality: QUALITY }); + case 'image/webp': + return instance.webp({ quality: QUALITY }); + case 'image/png': + // PNG is lossless, so quality does not apply and this will not shrink + // much. It still strips EXIF and still bounds the dimensions, which are + // the two things being bought here. + return instance.png({ compressionLevel: 9 }); + default: + // Unreachable: the allowlist in uploadTypes.ts is these three. Throwing + // rather than passing the file through unmodified, because "we did not + // recognise it so we left the metadata in" is the failure mode this + // module exists to make impossible. + throw new Error(`cannot re-encode unsupported type ${mimetype}`); + } +} + +/** + * Rewrites the file at `filePath`, in its own format, stripped and bounded. + * + * Writes to a sibling temporary file and renames over the original, because + * writing in place would leave a half-written image being served if the process + * died mid-write — and sharp cannot read and write the same path in one pass + * anyway. + * + * `withoutEnlargement` so a small image is not blown up to the cap: the ceiling + * is a maximum, not a target. + */ +export async function reencodeInPlace(filePath: string, mimetype: string): Promise { + const temporary = `${filePath}.reencoding`; + try { + await encoderFor( + // `animated` only for WebP, which is the one allowed type that can carry + // more than one frame. Reading an animated WebP without it decodes the + // first frame alone and silently writes back a still — destroying the + // uploader's image while reporting success. It is not set unconditionally + // because it changes how `resize` interprets height (the full frame + // strip, not one frame), which would be wrong for the other two. + sharp(filePath, mimetype === 'image/webp' ? { animated: true } : {}).resize({ + width: MAX_DIMENSION, + height: MAX_DIMENSION, + fit: 'inside', + withoutEnlargement: true + }), + mimetype + // No withMetadata(): omitting it is what drops EXIF, ICC and everything + // else. Calling it would put the metadata back. + ).toFile(temporary); + + await fs.rename(temporary, filePath); + } catch (err) { + await fs.unlink(temporary).catch(() => undefined); + throw err; + } +} diff --git a/backend/tests/unit/imageProcessing.test.ts b/backend/tests/unit/imageProcessing.test.ts new file mode 100644 index 0000000..849879a --- /dev/null +++ b/backend/tests/unit/imageProcessing.test.ts @@ -0,0 +1,31 @@ +import { needsProcessing, MAX_DIMENSION } from '../../src/imageProcessing'; + +// The whole of the skip/process policy, kept pure so the backfill's +// idempotency can be reasoned about without a filesystem. The backfill is +// lossy and irreversible, so being wrong here is expensive. +describe('needsProcessing', () => { + it('processes anything carrying EXIF, however small', () => { + expect(needsProcessing({ width: 10, height: 10, exif: Buffer.from('x') })).toBe(true); + }); + + it('processes an oversized image even with no EXIF', () => { + expect(needsProcessing({ width: MAX_DIMENSION + 1, height: 100 })).toBe(true); + }); + + it('processes an image oversized on either axis', () => { + expect(needsProcessing({ width: 100, height: MAX_DIMENSION + 1 })).toBe(true); + }); + + // The idempotency property the backfill depends on: a file already stripped + // and already within bounds is left alone, so a second run cannot put it + // through another lossy pass. + it('leaves a stripped, in-bounds image alone', () => { + expect(needsProcessing({ width: MAX_DIMENSION, height: MAX_DIMENSION })).toBe(false); + }); + + // Unknown dimensions mean sharp could not read it as an image. Processing is + // the safe answer: the alternative is skipping a file we understand least. + it('processes an image whose dimensions could not be read', () => { + expect(needsProcessing({})).toBe(true); + }); +}); From aecccef41817bbe9efdfc9d7c408ce6cc80dec5d Mon Sep 17 00:00:00 2001 From: Thom Lamb Date: Sat, 29 Aug 2026 11:54:34 -0500 Subject: [PATCH 3/4] feat(uploads): strip metadata from every accepted upload (#226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hooked into uploadImages rather than into the routes. That middleware is where verifyUploadedImages already runs and is the single choke point every upload path passes through, so the admin create and update routes are both covered and the intake route from #222 will inherit it rather than having to remember. The same reasoning discardUnlessAccepted already gives for being a hook instead of a call. Runs after verification, deliberately: re-encoding a file whose bytes do not match its declared type would be work on something already refused, and sharp's error would replace the clearer message that check produces. A re-encode failure refuses the upload rather than storing the original, because the one case where a photo keeps the coordinates it was taken at should not be the case nobody was told about. The test builds a JPEG carrying GPS tags rather than committing a binary fixture, so what it contains is readable, and it asserts the fixture really carries EXIF before asserting the stored file does not — otherwise the test would pass while proving nothing. GPS tags go in IFD3, which is the GPS IFD as libvips names it; sharp's Exif type has no separate GPS key, and putting them in IFD0 would have produced EXIF without producing the tags this issue is about. Backend suites: 285 unit, 260 integration, lint clean, build clean. One caveat worth recording. Across three full integration runs, `uploadValidation` failed once on "removes the upload when the request is refused for its other fields". It is a pre-existing race rather than a regression: discardUnlessAccepted cleans up in an unawaited `void discardUploads(...)` inside a `res.on('close')` handler, so a test asserting on the directory immediately after the response has always been able to observe the state before the unlink lands. Re-encoding adds enough libvips work to lose that race occasionally where it previously did not. The property still holds in production, where the process keeps running and the unlink completes. Filed separately rather than fixed here. Ref #226 --- backend/src/routes/admin.ts | 39 ++++++ .../exifStripping.integration.test.ts | 119 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 backend/tests/integration/exifStripping.integration.test.ts diff --git a/backend/src/routes/admin.ts b/backend/src/routes/admin.ts index 34a66ce..8f333a8 100755 --- a/backend/src/routes/admin.ts +++ b/backend/src/routes/admin.ts @@ -17,6 +17,7 @@ import { signatureMatches } from '../uploadTypes'; import { notifyFavoritersOfSale, notifyFavoritersOfRemoval, collectFavoriteRecipients } from '../favoriteAlerts'; +import { reencodeInPlace } from '../imageProcessing'; const router = Router(); @@ -178,6 +179,34 @@ async function verifyUploadedImages(req: Request): Promise { return null; } +/** + * Rebuilds every accepted file so it carries no metadata (#226). + * + * After verification, deliberately: re-encoding a file whose bytes do not match + * its declared type would be doing work on something already refused, and + * sharp's own error would replace the clearer message that check produces. + * + * A failure here refuses the upload rather than storing the original. Storing + * it would mean the one case where a photo keeps the coordinates it was taken + * at is the case nobody was told about. + * + * Returns the message to refuse with, or null when every file was rebuilt. + */ +async function stripUploadedImages(req: Request): Promise { + const files = (req.files as Express.Multer.File[]) || []; + + for (const file of files) { + try { + await reencodeInPlace(file.path, file.mimetype); + } catch (err) { + console.error(`[upload] could not re-encode ${file.path}:`, err); + return `${file.originalname} could not be processed`; + } + } + + return null; +} + // No error-handling middleware is mounted on the app, so translate multer's // limit errors here instead of letting them surface as a generic 500. const uploadImages = (req: Request, res: Response, next: NextFunction) => { @@ -200,6 +229,16 @@ const uploadImages = (req: Request, res: Response, next: NextFunction) => { verifyUploadedImages(req) .then((problem) => { + if (problem) { + res.status(400).json({ error: problem }); + return null; + } + return stripUploadedImages(req); + }) + .then((problem) => { + // The first stage returns null both when it answered and when it found + // nothing wrong, so the response itself is what distinguishes them. + if (res.headersSent) return; if (problem) { res.status(400).json({ error: problem }); return; diff --git a/backend/tests/integration/exifStripping.integration.test.ts b/backend/tests/integration/exifStripping.integration.test.ts new file mode 100644 index 0000000..6a529ed --- /dev/null +++ b/backend/tests/integration/exifStripping.integration.test.ts @@ -0,0 +1,119 @@ +import request from 'supertest'; +import sharp from 'sharp'; +import { promises as fs } from 'fs'; +import path from 'path'; +import app from '../../src/app'; +import { pool } from '../../src/db'; +import { resetDb, closeDb } from './setup/testDb'; + +const UPLOADS_DIR = process.env.UPLOADS_DIR as string; + +beforeAll(async () => { + await fs.mkdir(UPLOADS_DIR, { recursive: true }); +}); + +beforeEach(async () => { + await resetDb(); +}); + +afterAll(async () => { + await pool.end(); + await closeDb(); +}); + +/** + * A JPEG carrying GPS EXIF, built rather than committed as a binary fixture so + * what it contains is readable in this file. This is the exact shape of the + * problem: a photograph that says where it was taken. + */ +async function photoWithLocation(): Promise { + return sharp({ + create: { width: 3000, height: 2000, channels: 3, background: { r: 120, g: 90, b: 60 } } + }) + // GPS tags live in IFD3 — that is the GPS IFD as libvips names it, and + // sharp's Exif type has no separate `GPS` key. Writing them into IFD0 + // instead would still produce EXIF, but not the tags this issue is + // actually about. + .withExif({ + IFD0: { Make: 'TestCam', Model: 'X1' }, + IFD3: { GPSLatitudeRef: 'N', GPSLatitude: '51/1 30/1 0/1', GPSLongitudeRef: 'W' } + }) + .jpeg() + .toBuffer(); +} + +async function storedPathFor(itemId: number): Promise { + const { rows } = await pool.query<{ image_path: string }>( + `SELECT image_path FROM item_images WHERE item_id = $1 ORDER BY sort_order`, + [itemId] + ); + const row = rows[0]; + if (!row) throw new Error(`no item_images row for item ${itemId}`); + // image_path is '/uploads/'; the file is that name inside UPLOADS_DIR. + return path.join(UPLOADS_DIR, path.basename(row.image_path)); +} + +async function createItemWith(image: Buffer, filename: string): Promise { + const res = await request(app) + .post('/api/admin/items') + .field('name', 'Vase') + .field('description', '') + .field('price', '40') + .attach('images', image, filename); + + expect(res.status).toBe(200); + return res.body.id; +} + +describe('an uploaded photo does not keep where it was taken', () => { + it('has no EXIF once stored', async () => { + const withGps = await photoWithLocation(); + // Guard the fixture itself: if this ever stops carrying EXIF, the + // assertion below would pass while testing nothing at all. + expect((await sharp(withGps).metadata()).exif).toBeDefined(); + + const itemId = await createItemWith(withGps, 'vase.jpg'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.exif).toBeUndefined(); + }); + + it('is bounded to the maximum dimension', async () => { + const itemId = await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.width).toBe(2000); + expect(stored.height).toBe(1333); + }); + + it('keeps the format, so the stored extension still describes the file', async () => { + const itemId = await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const storedPath = await storedPathFor(itemId); + expect(path.extname(storedPath)).toBe('.jpg'); + expect((await sharp(storedPath).metadata()).format).toBe('jpeg'); + }); + + it('does not enlarge an image that is already small', async () => { + const small = await sharp({ + create: { width: 300, height: 200, channels: 3, background: { r: 1, g: 2, b: 3 } } + }) + .png() + .toBuffer(); + + const itemId = await createItemWith(small, 'tiny.png'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.width).toBe(300); + expect(stored.height).toBe(200); + }); + + it('leaves no temporary re-encoding files on the volume', async () => { + await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const leftovers = (await fs.readdir(UPLOADS_DIR)).filter((name) => + name.endsWith('.reencoding') + ); + expect(leftovers).toEqual([]); + }); +}); From 502d56d9fda62070aa418d5e740ff66ad2b9a8eb Mon Sep 17 00:00:00 2001 From: Thom Lamb Date: Sat, 29 Aug 2026 13:51:26 -0500 Subject: [PATCH 4/4] feat(uploads): backfill re-encoding over already-stored photos (#226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stripping new uploads does nothing for the catalogue that is already on the storefront, which is where the exposure actually lives today. This is the other half. Reports by default and rewrites nothing without --apply, because the transform is lossy and there is no undo. Idempotency comes from `needsProcessing` rather than from a marker or a schema change: a file with no EXIF already inside the bounds is already in its final state, so a second run skips it instead of putting it through another lossy pass. Proven rather than assumed — a second --apply immediately after the first reports skipped 1, processed 0. Verified end to end against a real row and a real file. 3000x2000 carrying GPS EXIF became 2000x1333 with the metadata gone, 35760 bytes down to 16019, the format preserved, no temporary file left behind, and `item_images.image_path` untouched. That last part is what preserving the format bought: the backfill rewrites bytes and writes nothing to the database, so there is no window where a row points at a file that no longer exists. Both degenerate branches are exercised too, since a script that dies partway through a catalogue leaves the rest of it exposed: a row pointing at a missing file and a row with an extension the application would refuse to serve are each reported and counted, and the run continues. `handleRow` is split out of `run` for cognitive complexity, and while doing that a miscount was introduced and caught — incrementing `processed` before the rewrite meant a file that threw would have been counted as both processed and failed, which makes the summary unreadable at the moment it matters most. Ref #226 --- backend/package.json | 1 + backend/scripts/backfill-image-reencode.ts | 160 +++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 backend/scripts/backfill-image-reencode.ts diff --git a/backend/package.json b/backend/package.json index 4d250b9..edf1c54 100755 --- a/backend/package.json +++ b/backend/package.json @@ -19,6 +19,7 @@ "test:integration:json": "jest -c jest.integration.config.js --runInBand --json --outputFile=integration-results.json", "test:integration:cov": "jest -c jest.integration.config.js --runInBand --coverage --forceExit", "bench:hashing": "tsx scripts/bench-hash-latency.ts", + "backfill:images": "tsx scripts/backfill-image-reencode.ts", "db:test:up": "docker compose -f docker-compose.test.yml up -d", "db:test:down": "docker compose -f docker-compose.test.yml down -v", "migrate:up": "node migrate.js up", diff --git a/backend/scripts/backfill-image-reencode.ts b/backend/scripts/backfill-image-reencode.ts new file mode 100644 index 0000000..c16b048 --- /dev/null +++ b/backend/scripts/backfill-image-reencode.ts @@ -0,0 +1,160 @@ +/** + * Applies #226's re-encoding to the photos that were stored before it existed. + * + * New uploads are handled in the request path. Everything already on the volume + * still carries whatever the camera wrote, including the coordinates the photo + * was taken at, and is still served publicly. This is the other half. + * + * The transform is lossy and there is no undo, so: + * + * - It reports by default and changes nothing without --apply. + * - It is idempotent. `needsProcessing` skips a file that is already stripped + * and already within bounds, so a second run is not a second lossy pass. + * - `reencodeInPlace` writes to a temporary file and renames, so an + * interruption cannot leave a half-written image being served. + * - It never renames the stored file, so item_images.image_path stays correct + * and no database write is needed at all. + * + * Usage + * ----- + * npm run backfill:images # report only + * npm run backfill:images -- --apply # rewrite the files + * + * Point it at QA first. Compare a handful of images by eye before production, + * and take a backup that you have confirmed restores. + */ + +import sharp from 'sharp'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { pool } from '../src/db'; +import { needsProcessing, reencodeInPlace } from '../src/imageProcessing'; + +const UPLOADS_DIR = process.env.UPLOADS_DIR || '/app/uploads'; +const APPLY = process.argv.includes('--apply'); + +// The stored extension is the file's real type — uploadTypes.ts derives it from +// the validated content type on the way in, so it can be trusted on the way +// back out. Anything else is a file this application would refuse to serve. +const TYPE_FOR_EXTENSION: Readonly> = { + '.jpg': 'image/jpeg', + '.png': 'image/png', + '.webp': 'image/webp' +}; + +interface Totals { + seen: number; + missing: number; + skipped: number; + unrecognised: number; + processed: number; + failed: number; + bytesBefore: number; + bytesAfter: number; +} + +/** + * One stored image: classify it, and rewrite it when it needs rewriting. + * + * Split out of `run` so that the loop reads as a loop. Every outcome is + * counted rather than thrown, because one unreadable file in a catalogue is + * not a reason to leave the rest of it exposed. + */ +async function handleRow(imagePath: string, totals: Totals): Promise { + // basename only: image_path is '/uploads/', and the directory it is + // served from is a server constant rather than part of the stored value. + const filePath = path.join(UPLOADS_DIR, path.basename(imagePath)); + const mimetype = TYPE_FOR_EXTENSION[path.extname(filePath).toLowerCase()]; + + if (!mimetype) { + console.warn(`[backfill] unrecognised extension, skipping: ${imagePath}`); + totals.unrecognised++; + return; + } + + let before: number; + try { + before = (await fs.stat(filePath)).size; + } catch { + // A row pointing at nothing is a pre-existing inconsistency. Reported + // rather than fatal: it is not this script's job to fix, and stopping + // would leave the rest of the catalogue exposed. + console.warn(`[backfill] file missing for ${imagePath}`); + totals.missing++; + return; + } + + try { + const meta = await sharp(filePath).metadata(); + + if (!needsProcessing(meta)) { + totals.skipped++; + return; + } + + if (!APPLY) { + console.info( + `[backfill] would process ${imagePath} ` + + `(${meta.width}x${meta.height}, exif ${meta.exif ? 'present' : 'absent'}, ${before} bytes)` + ); + totals.processed++; + totals.bytesBefore += before; + return; + } + + await reencodeInPlace(filePath, mimetype); + const after = (await fs.stat(filePath)).size; + // Counted only once the rewrite succeeded, so a file that threw is `failed` + // and nothing else. A row counted as both processed and failed would make + // the summary unreadable at exactly the moment it matters. + totals.processed++; + totals.bytesBefore += before; + totals.bytesAfter += after; + console.info(`[backfill] ${imagePath}: ${before} -> ${after} bytes`); + } catch (err) { + console.error(`[backfill] failed on ${imagePath}:`, err); + totals.failed++; + } +} + +async function run(): Promise { + const totals: Totals = { + seen: 0, + missing: 0, + skipped: 0, + unrecognised: 0, + processed: 0, + failed: 0, + bytesBefore: 0, + bytesAfter: 0 + }; + + const { rows } = await pool.query<{ image_path: string }>( + `SELECT image_path FROM item_images ORDER BY id` + ); + + console.info( + `[backfill] ${rows.length} image rows in ${UPLOADS_DIR}, ` + + `${APPLY ? 'APPLYING CHANGES' : 'reporting only (pass --apply to rewrite)'}` + ); + + for (const row of rows) { + totals.seen++; + await handleRow(row.image_path, totals); + } + + console.info('[backfill] done', totals); + + if (totals.failed > 0) { + // A non-zero exit so a partial run is visible to whatever invoked it, + // rather than reading as success because the summary printed. + process.exitCode = 1; + } +} + +run() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => pool.end());