diff --git a/README.md b/README.md index a22d78e..f1b308b 100755 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ Both scripts run `nvm use latest` first and verify the result is Node 20 or newe The Node 20 floor is not arbitrary: `node-pg-migrate` pulls in an `lru-cache` that calls `diagnostics_channel.tracingChannel()`, which does not exist before Node 19.9. On Node 18 migrations die inside minified library code with `(0 , U.tracingChannel) is not a function`, which says nothing about versions. +Since #226 there is a second Node floor, and it bites at **install** time rather than at run time. `sharp` declares `>=20.9.0`, and the platform binary that does its actual work is an **optional** dependency. npm silently skips an optional dependency whose engine check fails and still reports success — so `npm install` on the machine's default 18.16.1 produces a `node_modules` that looks complete and then throws `Could not load the "sharp" module using the win32-x64 runtime` at require time. That message names a runtime rather than a version and sends you looking in the wrong place. + +Once the binary is installed, sharp loads and runs perfectly well on 18.16.1 — `engines` is advisory at run time. So this is purely about how the install was done, not about which Node runs the tests. Install through `start-local.ps1` or `run-tests.ps1`, which switch to Node 20+ first; if you have already hit it, `npm install --include=optional sharp` under Node 20+ repairs it in place. + `run-tests.ps1` brings up whatever a suite needs: the integration suite gets its own throwaway Postgres, started and stopped around the run (`-KeepTestDb` leaves it up, `-TestDbPort` moves it if the default is taken or Hyper-V has reserved it). The e2e suite needs the app stack, so start it with `start-local.ps1` first — the script checks and says so rather than letting every spec fail on a refused connection. `-Filter` passes through to the runner to select tests by file or name. ### 1. Start a local Postgres instance diff --git a/backend/package-lock.json b/backend/package-lock.json index a9b0d05..59b06f3 100755 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -19,7 +19,8 @@ "node-cron": "^3.0.3", "node-pg-migrate": "^7.6.1", "nodemailer": "^6.9.14", - "pg": "^8.12.0" + "pg": "^8.12.0", + "sharp": "^0.35.4" }, "devDependencies": { "@eslint/js": "^9.39.5", @@ -46,6 +47,9 @@ "tsx": "^4.16.5", "typescript": "^5.5.4", "typescript-eslint": "^8.67.0" + }, + "engines": { + "node": ">=20.9.0" } }, "node_modules/@babel/code-frame": { @@ -1827,6 +1831,516 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", + "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", + "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", + "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", + "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", + "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", + "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", + "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", + "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", + "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", + "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", + "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", + "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", + "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", + "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", + "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", + "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", + "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", + "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.3" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", + "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", + "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", + "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.3" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", + "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-wasm32/node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", + "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", + "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", + "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", + "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", @@ -2366,9 +2880,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2386,9 +2897,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2406,9 +2914,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2426,9 +2931,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2446,9 +2948,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2466,9 +2965,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2486,9 +2982,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2506,9 +2999,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4271,6 +4761,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8535,6 +9034,67 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.35.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", + "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.4", + "@img/sharp-darwin-x64": "0.35.4", + "@img/sharp-freebsd-wasm32": "0.35.4", + "@img/sharp-libvips-darwin-arm64": "1.3.3", + "@img/sharp-libvips-darwin-x64": "1.3.3", + "@img/sharp-libvips-linux-arm": "1.3.3", + "@img/sharp-libvips-linux-arm64": "1.3.3", + "@img/sharp-libvips-linux-ppc64": "1.3.3", + "@img/sharp-libvips-linux-riscv64": "1.3.3", + "@img/sharp-libvips-linux-s390x": "1.3.3", + "@img/sharp-libvips-linux-x64": "1.3.3", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", + "@img/sharp-libvips-linuxmusl-x64": "1.3.3", + "@img/sharp-linux-arm": "0.35.4", + "@img/sharp-linux-arm64": "0.35.4", + "@img/sharp-linux-ppc64": "0.35.4", + "@img/sharp-linux-riscv64": "0.35.4", + "@img/sharp-linux-s390x": "0.35.4", + "@img/sharp-linux-x64": "0.35.4", + "@img/sharp-linuxmusl-arm64": "0.35.4", + "@img/sharp-linuxmusl-x64": "0.35.4", + "@img/sharp-webcontainers-wasm32": "0.35.4", + "@img/sharp-win32-arm64": "0.35.4", + "@img/sharp-win32-ia32": "0.35.4", + "@img/sharp-win32-x64": "0.35.4" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -9115,7 +9675,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD", "optional": true }, diff --git a/backend/package.json b/backend/package.json index 7b2686e..2024a22 100755 --- a/backend/package.json +++ b/backend/package.json @@ -3,6 +3,9 @@ "version": "1.0.0", "private": true, "main": "dist/server.js", + "engines": { + "node": ">=20.9.0" + }, "scripts": { "build": "tsc", "lint": "eslint src scripts", @@ -16,6 +19,7 @@ "test:integration:json": "jest -c jest.integration.config.js --runInBand --json --outputFile=integration-results.json", "test:integration:cov": "jest -c jest.integration.config.js --runInBand --coverage --forceExit", "bench:hashing": "tsx scripts/bench-hash-latency.ts", + "backfill:images": "tsx scripts/backfill-image-reencode.ts", "db:test:up": "docker compose -f docker-compose.test.yml up -d", "db:test:down": "docker compose -f docker-compose.test.yml down -v", "migrate:up": "node migrate.js up", @@ -34,7 +38,8 @@ "node-cron": "^3.0.3", "node-pg-migrate": "^7.6.1", "nodemailer": "^6.9.14", - "pg": "^8.12.0" + "pg": "^8.12.0", + "sharp": "^0.35.4" }, "devDependencies": { "@eslint/js": "^9.39.5", diff --git a/backend/scripts/backfill-image-reencode.ts b/backend/scripts/backfill-image-reencode.ts new file mode 100644 index 0000000..c16b048 --- /dev/null +++ b/backend/scripts/backfill-image-reencode.ts @@ -0,0 +1,160 @@ +/** + * Applies #226's re-encoding to the photos that were stored before it existed. + * + * New uploads are handled in the request path. Everything already on the volume + * still carries whatever the camera wrote, including the coordinates the photo + * was taken at, and is still served publicly. This is the other half. + * + * The transform is lossy and there is no undo, so: + * + * - It reports by default and changes nothing without --apply. + * - It is idempotent. `needsProcessing` skips a file that is already stripped + * and already within bounds, so a second run is not a second lossy pass. + * - `reencodeInPlace` writes to a temporary file and renames, so an + * interruption cannot leave a half-written image being served. + * - It never renames the stored file, so item_images.image_path stays correct + * and no database write is needed at all. + * + * Usage + * ----- + * npm run backfill:images # report only + * npm run backfill:images -- --apply # rewrite the files + * + * Point it at QA first. Compare a handful of images by eye before production, + * and take a backup that you have confirmed restores. + */ + +import sharp from 'sharp'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { pool } from '../src/db'; +import { needsProcessing, reencodeInPlace } from '../src/imageProcessing'; + +const UPLOADS_DIR = process.env.UPLOADS_DIR || '/app/uploads'; +const APPLY = process.argv.includes('--apply'); + +// The stored extension is the file's real type — uploadTypes.ts derives it from +// the validated content type on the way in, so it can be trusted on the way +// back out. Anything else is a file this application would refuse to serve. +const TYPE_FOR_EXTENSION: Readonly> = { + '.jpg': 'image/jpeg', + '.png': 'image/png', + '.webp': 'image/webp' +}; + +interface Totals { + seen: number; + missing: number; + skipped: number; + unrecognised: number; + processed: number; + failed: number; + bytesBefore: number; + bytesAfter: number; +} + +/** + * One stored image: classify it, and rewrite it when it needs rewriting. + * + * Split out of `run` so that the loop reads as a loop. Every outcome is + * counted rather than thrown, because one unreadable file in a catalogue is + * not a reason to leave the rest of it exposed. + */ +async function handleRow(imagePath: string, totals: Totals): Promise { + // basename only: image_path is '/uploads/', and the directory it is + // served from is a server constant rather than part of the stored value. + const filePath = path.join(UPLOADS_DIR, path.basename(imagePath)); + const mimetype = TYPE_FOR_EXTENSION[path.extname(filePath).toLowerCase()]; + + if (!mimetype) { + console.warn(`[backfill] unrecognised extension, skipping: ${imagePath}`); + totals.unrecognised++; + return; + } + + let before: number; + try { + before = (await fs.stat(filePath)).size; + } catch { + // A row pointing at nothing is a pre-existing inconsistency. Reported + // rather than fatal: it is not this script's job to fix, and stopping + // would leave the rest of the catalogue exposed. + console.warn(`[backfill] file missing for ${imagePath}`); + totals.missing++; + return; + } + + try { + const meta = await sharp(filePath).metadata(); + + if (!needsProcessing(meta)) { + totals.skipped++; + return; + } + + if (!APPLY) { + console.info( + `[backfill] would process ${imagePath} ` + + `(${meta.width}x${meta.height}, exif ${meta.exif ? 'present' : 'absent'}, ${before} bytes)` + ); + totals.processed++; + totals.bytesBefore += before; + return; + } + + await reencodeInPlace(filePath, mimetype); + const after = (await fs.stat(filePath)).size; + // Counted only once the rewrite succeeded, so a file that threw is `failed` + // and nothing else. A row counted as both processed and failed would make + // the summary unreadable at exactly the moment it matters. + totals.processed++; + totals.bytesBefore += before; + totals.bytesAfter += after; + console.info(`[backfill] ${imagePath}: ${before} -> ${after} bytes`); + } catch (err) { + console.error(`[backfill] failed on ${imagePath}:`, err); + totals.failed++; + } +} + +async function run(): Promise { + const totals: Totals = { + seen: 0, + missing: 0, + skipped: 0, + unrecognised: 0, + processed: 0, + failed: 0, + bytesBefore: 0, + bytesAfter: 0 + }; + + const { rows } = await pool.query<{ image_path: string }>( + `SELECT image_path FROM item_images ORDER BY id` + ); + + console.info( + `[backfill] ${rows.length} image rows in ${UPLOADS_DIR}, ` + + `${APPLY ? 'APPLYING CHANGES' : 'reporting only (pass --apply to rewrite)'}` + ); + + for (const row of rows) { + totals.seen++; + await handleRow(row.image_path, totals); + } + + console.info('[backfill] done', totals); + + if (totals.failed > 0) { + // A non-zero exit so a partial run is visible to whatever invoked it, + // rather than reading as success because the summary printed. + process.exitCode = 1; + } +} + +run() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(() => pool.end()); diff --git a/backend/src/imageProcessing.ts b/backend/src/imageProcessing.ts new file mode 100644 index 0000000..0314fb2 --- /dev/null +++ b/backend/src/imageProcessing.ts @@ -0,0 +1,108 @@ +import sharp, { type Sharp } from 'sharp'; +import { promises as fs } from 'fs'; + +/** + * Rebuilding an uploaded image so it carries nothing but the picture. + * + * A file arrives as the camera wrote it, and a camera writes EXIF — which + * routinely includes the coordinates the photo was taken at. Those files are + * served publicly from /uploads/, so an unmodified product photo publishes the + * location it was taken. Nobody sending in a photograph of a vase expects that. + * + * The fix is to re-encode rather than to delete tags. Deleting requires knowing + * every tag that could carry something sensitive, across formats and camera + * makers, forever. Re-encoding builds a new file from the decoded pixels, so + * there is nothing left that could have been missed — the same reasoning that + * makes uploadTypes.ts an allowlist rather than a denylist. + * + * Format is deliberately preserved. Converting to WebP would compress better, + * but it changes stored extensions, and therefore item_images.image_path, and + * therefore turns the backfill into a rename with a window where rows point at + * files that no longer exist. See #226. + */ + +/** Comfortably larger than anything the storefront renders. */ +export const MAX_DIMENSION = 2000; + +/** Where further reduction starts to show on a photograph. */ +export const QUALITY = 82; + +interface ImageFacts { + width?: number; + height?: number; + exif?: unknown; +} + +/** + * Whether a file still needs rebuilding. + * + * Pure, and the backfill's entire idempotency argument: a file with no EXIF + * that is already within bounds is already in its final state, so a re-run + * skips it rather than putting it through a second lossy pass. Anything + * unreadable is processed rather than skipped — a file we cannot describe is + * not one to assume is safe. + */ +export function needsProcessing(meta: ImageFacts): boolean { + if (meta.exif !== undefined && meta.exif !== null) return true; + if (meta.width === undefined || meta.height === undefined) return true; + return meta.width > MAX_DIMENSION || meta.height > MAX_DIMENSION; +} + +function encoderFor(instance: Sharp, mimetype: string): Sharp { + switch (mimetype) { + case 'image/jpeg': + return instance.jpeg({ quality: QUALITY }); + case 'image/webp': + return instance.webp({ quality: QUALITY }); + case 'image/png': + // PNG is lossless, so quality does not apply and this will not shrink + // much. It still strips EXIF and still bounds the dimensions, which are + // the two things being bought here. + return instance.png({ compressionLevel: 9 }); + default: + // Unreachable: the allowlist in uploadTypes.ts is these three. Throwing + // rather than passing the file through unmodified, because "we did not + // recognise it so we left the metadata in" is the failure mode this + // module exists to make impossible. + throw new Error(`cannot re-encode unsupported type ${mimetype}`); + } +} + +/** + * Rewrites the file at `filePath`, in its own format, stripped and bounded. + * + * Writes to a sibling temporary file and renames over the original, because + * writing in place would leave a half-written image being served if the process + * died mid-write — and sharp cannot read and write the same path in one pass + * anyway. + * + * `withoutEnlargement` so a small image is not blown up to the cap: the ceiling + * is a maximum, not a target. + */ +export async function reencodeInPlace(filePath: string, mimetype: string): Promise { + const temporary = `${filePath}.reencoding`; + try { + await encoderFor( + // `animated` only for WebP, which is the one allowed type that can carry + // more than one frame. Reading an animated WebP without it decodes the + // first frame alone and silently writes back a still — destroying the + // uploader's image while reporting success. It is not set unconditionally + // because it changes how `resize` interprets height (the full frame + // strip, not one frame), which would be wrong for the other two. + sharp(filePath, mimetype === 'image/webp' ? { animated: true } : {}).resize({ + width: MAX_DIMENSION, + height: MAX_DIMENSION, + fit: 'inside', + withoutEnlargement: true + }), + mimetype + // No withMetadata(): omitting it is what drops EXIF, ICC and everything + // else. Calling it would put the metadata back. + ).toFile(temporary); + + await fs.rename(temporary, filePath); + } catch (err) { + await fs.unlink(temporary).catch(() => undefined); + throw err; + } +} diff --git a/backend/src/routes/admin.ts b/backend/src/routes/admin.ts index 34a66ce..8f333a8 100755 --- a/backend/src/routes/admin.ts +++ b/backend/src/routes/admin.ts @@ -17,6 +17,7 @@ import { signatureMatches } from '../uploadTypes'; import { notifyFavoritersOfSale, notifyFavoritersOfRemoval, collectFavoriteRecipients } from '../favoriteAlerts'; +import { reencodeInPlace } from '../imageProcessing'; const router = Router(); @@ -178,6 +179,34 @@ async function verifyUploadedImages(req: Request): Promise { return null; } +/** + * Rebuilds every accepted file so it carries no metadata (#226). + * + * After verification, deliberately: re-encoding a file whose bytes do not match + * its declared type would be doing work on something already refused, and + * sharp's own error would replace the clearer message that check produces. + * + * A failure here refuses the upload rather than storing the original. Storing + * it would mean the one case where a photo keeps the coordinates it was taken + * at is the case nobody was told about. + * + * Returns the message to refuse with, or null when every file was rebuilt. + */ +async function stripUploadedImages(req: Request): Promise { + const files = (req.files as Express.Multer.File[]) || []; + + for (const file of files) { + try { + await reencodeInPlace(file.path, file.mimetype); + } catch (err) { + console.error(`[upload] could not re-encode ${file.path}:`, err); + return `${file.originalname} could not be processed`; + } + } + + return null; +} + // No error-handling middleware is mounted on the app, so translate multer's // limit errors here instead of letting them surface as a generic 500. const uploadImages = (req: Request, res: Response, next: NextFunction) => { @@ -200,6 +229,16 @@ const uploadImages = (req: Request, res: Response, next: NextFunction) => { verifyUploadedImages(req) .then((problem) => { + if (problem) { + res.status(400).json({ error: problem }); + return null; + } + return stripUploadedImages(req); + }) + .then((problem) => { + // The first stage returns null both when it answered and when it found + // nothing wrong, so the response itself is what distinguishes them. + if (res.headersSent) return; if (problem) { res.status(400).json({ error: problem }); return; diff --git a/backend/tests/integration/exifStripping.integration.test.ts b/backend/tests/integration/exifStripping.integration.test.ts new file mode 100644 index 0000000..6a529ed --- /dev/null +++ b/backend/tests/integration/exifStripping.integration.test.ts @@ -0,0 +1,119 @@ +import request from 'supertest'; +import sharp from 'sharp'; +import { promises as fs } from 'fs'; +import path from 'path'; +import app from '../../src/app'; +import { pool } from '../../src/db'; +import { resetDb, closeDb } from './setup/testDb'; + +const UPLOADS_DIR = process.env.UPLOADS_DIR as string; + +beforeAll(async () => { + await fs.mkdir(UPLOADS_DIR, { recursive: true }); +}); + +beforeEach(async () => { + await resetDb(); +}); + +afterAll(async () => { + await pool.end(); + await closeDb(); +}); + +/** + * A JPEG carrying GPS EXIF, built rather than committed as a binary fixture so + * what it contains is readable in this file. This is the exact shape of the + * problem: a photograph that says where it was taken. + */ +async function photoWithLocation(): Promise { + return sharp({ + create: { width: 3000, height: 2000, channels: 3, background: { r: 120, g: 90, b: 60 } } + }) + // GPS tags live in IFD3 — that is the GPS IFD as libvips names it, and + // sharp's Exif type has no separate `GPS` key. Writing them into IFD0 + // instead would still produce EXIF, but not the tags this issue is + // actually about. + .withExif({ + IFD0: { Make: 'TestCam', Model: 'X1' }, + IFD3: { GPSLatitudeRef: 'N', GPSLatitude: '51/1 30/1 0/1', GPSLongitudeRef: 'W' } + }) + .jpeg() + .toBuffer(); +} + +async function storedPathFor(itemId: number): Promise { + const { rows } = await pool.query<{ image_path: string }>( + `SELECT image_path FROM item_images WHERE item_id = $1 ORDER BY sort_order`, + [itemId] + ); + const row = rows[0]; + if (!row) throw new Error(`no item_images row for item ${itemId}`); + // image_path is '/uploads/'; the file is that name inside UPLOADS_DIR. + return path.join(UPLOADS_DIR, path.basename(row.image_path)); +} + +async function createItemWith(image: Buffer, filename: string): Promise { + const res = await request(app) + .post('/api/admin/items') + .field('name', 'Vase') + .field('description', '') + .field('price', '40') + .attach('images', image, filename); + + expect(res.status).toBe(200); + return res.body.id; +} + +describe('an uploaded photo does not keep where it was taken', () => { + it('has no EXIF once stored', async () => { + const withGps = await photoWithLocation(); + // Guard the fixture itself: if this ever stops carrying EXIF, the + // assertion below would pass while testing nothing at all. + expect((await sharp(withGps).metadata()).exif).toBeDefined(); + + const itemId = await createItemWith(withGps, 'vase.jpg'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.exif).toBeUndefined(); + }); + + it('is bounded to the maximum dimension', async () => { + const itemId = await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.width).toBe(2000); + expect(stored.height).toBe(1333); + }); + + it('keeps the format, so the stored extension still describes the file', async () => { + const itemId = await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const storedPath = await storedPathFor(itemId); + expect(path.extname(storedPath)).toBe('.jpg'); + expect((await sharp(storedPath).metadata()).format).toBe('jpeg'); + }); + + it('does not enlarge an image that is already small', async () => { + const small = await sharp({ + create: { width: 300, height: 200, channels: 3, background: { r: 1, g: 2, b: 3 } } + }) + .png() + .toBuffer(); + + const itemId = await createItemWith(small, 'tiny.png'); + + const stored = await sharp(await storedPathFor(itemId)).metadata(); + expect(stored.width).toBe(300); + expect(stored.height).toBe(200); + }); + + it('leaves no temporary re-encoding files on the volume', async () => { + await createItemWith(await photoWithLocation(), 'vase.jpg'); + + const leftovers = (await fs.readdir(UPLOADS_DIR)).filter((name) => + name.endsWith('.reencoding') + ); + expect(leftovers).toEqual([]); + }); +}); diff --git a/backend/tests/unit/imageProcessing.test.ts b/backend/tests/unit/imageProcessing.test.ts new file mode 100644 index 0000000..849879a --- /dev/null +++ b/backend/tests/unit/imageProcessing.test.ts @@ -0,0 +1,31 @@ +import { needsProcessing, MAX_DIMENSION } from '../../src/imageProcessing'; + +// The whole of the skip/process policy, kept pure so the backfill's +// idempotency can be reasoned about without a filesystem. The backfill is +// lossy and irreversible, so being wrong here is expensive. +describe('needsProcessing', () => { + it('processes anything carrying EXIF, however small', () => { + expect(needsProcessing({ width: 10, height: 10, exif: Buffer.from('x') })).toBe(true); + }); + + it('processes an oversized image even with no EXIF', () => { + expect(needsProcessing({ width: MAX_DIMENSION + 1, height: 100 })).toBe(true); + }); + + it('processes an image oversized on either axis', () => { + expect(needsProcessing({ width: 100, height: MAX_DIMENSION + 1 })).toBe(true); + }); + + // The idempotency property the backfill depends on: a file already stripped + // and already within bounds is left alone, so a second run cannot put it + // through another lossy pass. + it('leaves a stripped, in-bounds image alone', () => { + expect(needsProcessing({ width: MAX_DIMENSION, height: MAX_DIMENSION })).toBe(false); + }); + + // Unknown dimensions mean sharp could not read it as an image. Processing is + // the safe answer: the alternative is skipping a file we understand least. + it('processes an image whose dimensions could not be read', () => { + expect(needsProcessing({})).toBe(true); + }); +});