From 063beea2cada511cd44a9374a269474e0ea91b92 Mon Sep 17 00:00:00 2001 From: Ryan Cohen Date: Fri, 30 Sep 2022 15:34:51 +0900 Subject: [PATCH] feat: custom image proxy --- .gitignore | 3 + server/index.ts | 4 + server/lib/imageproxy.ts | 185 ++++++++++++++++++ server/routes/imageproxy.ts | 39 ++++ src/components/Common/CachedImage/index.tsx | 18 +- src/components/CompanyCard/index.tsx | 15 +- .../MediaSlider/ShowMoreCard/index.tsx | 14 ++ 7 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 server/lib/imageproxy.ts create mode 100644 server/routes/imageproxy.ts diff --git a/.gitignore b/.gitignore index 4f7c3ce6..6e051447 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ cypress/screenshots # TS Build Info tsconfig.tsbuildinfo + +# Config Cache Directory +config/cache diff --git a/server/index.ts b/server/index.ts index e78d5a57..12df2f1f 100644 --- a/server/index.ts +++ b/server/index.ts @@ -17,6 +17,7 @@ import WebPushAgent from '@server/lib/notifications/agents/webpush'; import { getSettings } from '@server/lib/settings'; import logger from '@server/logger'; import routes from '@server/routes'; +import imageproxy from '@server/routes/imageproxy'; import { getAppVersion } from '@server/utils/appVersion'; import restartFlag from '@server/utils/restartFlag'; import { getClientIp } from '@supercharge/request-ip'; @@ -176,6 +177,9 @@ app next(); }); server.use('/api/v1', routes); + + server.use('/imageproxy', imageproxy); + server.get('*', (req, res) => handle(req, res)); server.use( ( diff --git a/server/lib/imageproxy.ts b/server/lib/imageproxy.ts new file mode 100644 index 00000000..1b8aa21f --- /dev/null +++ b/server/lib/imageproxy.ts @@ -0,0 +1,185 @@ +import logger from '@server/logger'; +import axios from 'axios'; +import rateLimit, { type rateLimitOptions } from 'axios-rate-limit'; +import { createHash } from 'crypto'; +import { promises } from 'fs'; +import path, { join } from 'path'; + +type ImageResponse = { + meta: { + revalidateAfter: number; + curRevalidate: number; + isStale: boolean; + etag: string; + extension: string; + cacheKey: string; + cacheMiss: boolean; + }; + imageBuffer: Buffer; +}; + +class ImageProxy { + private axios; + private cacheVersion; + private key; + + constructor( + key: string, + baseUrl: string, + options: { + cacheVersion?: number; + rateLimitOptions?: rateLimitOptions; + } = {} + ) { + this.cacheVersion = options.cacheVersion ?? 1; + this.key = key; + this.axios = axios.create({ + baseURL: baseUrl, + }); + + if (options.rateLimitOptions) { + this.axios = rateLimit(this.axios, options.rateLimitOptions); + } + } + + public async getImage(path: string): Promise { + const cacheKey = this.getCacheKey(path); + + const imageResponse = await this.get(cacheKey); + + if (!imageResponse) { + const newImage = await this.set(path, cacheKey); + + if (!newImage) { + throw new Error('Failed to load image'); + } + + return newImage; + } + + // If the image is stale, we will revalidate it in the background. + if (imageResponse.meta.isStale) { + this.set(path, cacheKey); + } + + return imageResponse; + } + + private async get(cacheKey: string): Promise { + try { + const directory = join(this.getCacheDirectory(), cacheKey); + const files = await promises.readdir(directory); + const now = Date.now(); + + for (const file of files) { + const [maxAgeSt, expireAtSt, etag, extension] = file.split('.'); + const buffer = await promises.readFile(join(directory, file)); + const expireAt = Number(expireAtSt); + const maxAge = Number(maxAgeSt); + + return { + meta: { + curRevalidate: maxAge, + revalidateAfter: maxAge * 1000 + now, + isStale: now > expireAt, + etag, + extension, + cacheKey, + cacheMiss: false, + }, + imageBuffer: buffer, + }; + } + } catch (e) { + // No files. Treat as empty cache. + } + + return null; + } + + private async set( + path: string, + cacheKey: string + ): Promise { + try { + const directory = join(this.getCacheDirectory(), cacheKey); + const response = await this.axios.get(path, { + responseType: 'arraybuffer', + }); + + const buffer = Buffer.from(response.data, 'binary'); + const extension = path.split('.').pop() ?? ''; + const maxAge = Number(response.headers['cache-control'].split('=')[1]); + const expireAt = Date.now() + maxAge * 1000; + const etag = response.headers.etag.replace(/"/g, ''); + + await this.writeToCacheDir( + directory, + extension, + maxAge, + expireAt, + buffer, + etag + ); + + return { + meta: { + curRevalidate: maxAge, + revalidateAfter: expireAt, + isStale: false, + etag, + extension, + cacheKey, + cacheMiss: true, + }, + imageBuffer: buffer, + }; + } catch (e) { + logger.debug('Something went wrong caching image.', { + label: 'Image Cache', + errorMessage: e.message, + }); + return null; + } + } + + private async writeToCacheDir( + dir: string, + extension: string, + maxAge: number, + expireAt: number, + buffer: Buffer, + etag: string + ) { + const filename = join(dir, `${maxAge}.${expireAt}.${etag}.${extension}`); + + await promises.rm(dir, { force: true, recursive: true }).catch(() => { + // do nothing + }); + + await promises.mkdir(dir, { recursive: true }); + await promises.writeFile(filename, buffer); + } + + private getCacheKey(path: string) { + return this.getHash([this.key, this.cacheVersion, path]); + } + + private getHash(items: (string | number | Buffer)[]) { + const hash = createHash('sha256'); + for (const item of items) { + if (typeof item === 'number') hash.update(String(item)); + else { + hash.update(item); + } + } + // See https://en.wikipedia.org/wiki/Base64#Filenames + return hash.digest('base64').replace(/\//g, '-'); + } + + private getCacheDirectory() { + return path.join(__dirname, '../../config/cache/images/', this.key); + } +} + +export default ImageProxy; diff --git a/server/routes/imageproxy.ts b/server/routes/imageproxy.ts new file mode 100644 index 00000000..6cf104f5 --- /dev/null +++ b/server/routes/imageproxy.ts @@ -0,0 +1,39 @@ +import ImageProxy from '@server/lib/imageproxy'; +import logger from '@server/logger'; +import { Router } from 'express'; + +const router = Router(); +const tmdbImageProxy = new ImageProxy('tmdb', 'https://image.tmdb.org', { + rateLimitOptions: { + maxRequests: 20, + maxRPS: 50, + }, +}); + +/** + * Image Proxy + */ +router.get('/*', async (req, res) => { + const imagePath = req.path.replace('/image', ''); + try { + const imageData = await tmdbImageProxy.getImage(imagePath); + + res.writeHead(200, { + 'Content-Type': `image/${imageData.meta.extension}`, + 'Content-Length': imageData.imageBuffer.length, + 'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`, + 'OS-Cache-Key': imageData.meta.cacheKey, + 'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT', + }); + + res.end(imageData.imageBuffer); + } catch (e) { + logger.error('Failed to proxy image', { + imagePath, + errorMessage: e.message, + }); + res.status(500).send(); + } +}); + +export default router; diff --git a/src/components/Common/CachedImage/index.tsx b/src/components/Common/CachedImage/index.tsx index b1695937..3b1a67f4 100644 --- a/src/components/Common/CachedImage/index.tsx +++ b/src/components/Common/CachedImage/index.tsx @@ -1,7 +1,9 @@ import useSettings from '@app/hooks/useSettings'; -import type { ImageProps } from 'next/image'; +import type { ImageLoader, ImageProps } from 'next/image'; import Image from 'next/image'; +const imageLoader: ImageLoader = ({ src }) => src; + /** * The CachedImage component should be used wherever * we want to offer the option to locally cache images. @@ -9,10 +11,20 @@ import Image from 'next/image'; * It uses the `next/image` Image component but overrides * the `unoptimized` prop based on the application setting `cacheImages`. **/ -const CachedImage = (props: ImageProps) => { +const CachedImage = ({ src, ...props }: ImageProps) => { const { currentSettings } = useSettings(); - return ; + let imageUrl = src; + + if ( + typeof imageUrl === 'string' && + imageUrl.startsWith('https://image.tmdb.org') && + currentSettings.cacheImages + ) { + imageUrl = imageUrl.replace('https://image.tmdb.org', '/imageproxy'); + } + + return ; }; export default CachedImage; diff --git a/src/components/CompanyCard/index.tsx b/src/components/CompanyCard/index.tsx index 762d1a08..13b92a33 100644 --- a/src/components/CompanyCard/index.tsx +++ b/src/components/CompanyCard/index.tsx @@ -1,3 +1,4 @@ +import CachedImage from '@app/components/Common/CachedImage'; import Link from 'next/link'; import { useState } from 'react'; @@ -30,11 +31,15 @@ const CompanyCard = ({ image, url, name }: CompanyCardProps) => { role="link" tabIndex={0} > - {name} +
+ +
{ const intl = useIntl(); const [isHovered, setHovered] = useState(false); + const { ref, inView } = useInView({ + triggerOnce: true, + }); + + if (!inView) { + return ( +
+ +
+ ); + } + return (