merge: Add separate redis for rate limit (!908)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/908

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>

.config/ci.yml

@@ -103,6 +103,16 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForRateLimit:
+#  host: localhost
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+#  # You can specify more ioredis options...
+#  #username: example-username
+
 #   ┌───────────────────────────────┐
 #───┘ Fulltext search configuration └─────────────────────────────
 

.config/cypress-devcontainer.yml

@@ -124,6 +124,16 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForRateLimit:
+#  host: localhost
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+#  # You can specify more ioredis options...
+#  #username: example-username
+
 #   ┌───────────────────────────────┐
 #───┘ Fulltext search configuration └─────────────────────────────
 

.config/docker_example.yml

@@ -171,6 +171,16 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForRateLimit:
+#  host: localhost
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+#  # You can specify more ioredis options...
+#  #username: example-username
+
 #   ┌───────────────────────────────┐
 #───┘ Fulltext search configuration └─────────────────────────────
 

.config/example.yml

@@ -198,6 +198,16 @@ redis:
 #  # You can specify more ioredis options...
 #  #username: example-username
 
+#redisForRateLimit:
+#  host: localhost
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+#  # You can specify more ioredis options...
+#  #username: example-username
+
 #   ┌───────────────────────────────┐
 #───┘ Fulltext search configuration └─────────────────────────────
 

packages/backend/scripts/check_connect.js

@@ -51,6 +51,7 @@ const promises = Array
 		config.redisForJobQueue,
 		config.redisForTimelines,
 		config.redisForReactions,
+		config.redisForRateLimit,
 	]))
 	.map(connectToRedis)
 	.concat([

packages/backend/src/GlobalModule.ts

@@ -92,6 +92,14 @@ const $redisForReactions: Provider = {
 	inject: [DI.config],
 };
 
+const $redisForRateLimit: Provider = {
+	provide: DI.redisForRateLimit,
+	useFactory: (config: Config) => {
+		return new Redis.Redis(config.redisForRateLimit);
+	},
+	inject: [DI.config],
+};
+
 const $meta: Provider = {
 	provide: DI.meta,
 	useFactory: async (db: DataSource, redisForSub: Redis.Redis) => {
@@ -152,8 +160,8 @@ const $meta: Provider = {
 @Global()
 @Module({
 	imports: [RepositoryModule],
-	providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions],
+	providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit],
-	exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule],
+	exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, RepositoryModule],
 })
 export class GlobalModule implements OnApplicationShutdown {
 	constructor(
@@ -163,6 +171,7 @@ export class GlobalModule implements OnApplicationShutdown {
 		@Inject(DI.redisForSub) private redisForSub: Redis.Redis,
 		@Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis,
 		@Inject(DI.redisForReactions) private redisForReactions: Redis.Redis,
+		@Inject(DI.redisForRateLimit) private redisForRateLimit: Redis.Redis,
 	) { }
 
 	public async dispose(): Promise<void> {
@@ -176,6 +185,7 @@ export class GlobalModule implements OnApplicationShutdown {
 			this.redisForSub.disconnect(),
 			this.redisForTimelines.disconnect(),
 			this.redisForReactions.disconnect(),
+			this.redisForRateLimit.disconnect(),
 		]);
 	}
 

packages/backend/src/config.ts

@@ -53,6 +53,7 @@ type Source = {
 	redisForJobQueue?: RedisOptionsSource;
 	redisForTimelines?: RedisOptionsSource;
 	redisForReactions?: RedisOptionsSource;
+	redisForRateLimit?: RedisOptionsSource;
 	fulltextSearch?: {
 		provider?: FulltextSearchProvider;
 	};
@@ -231,6 +232,7 @@ export type Config = {
 	redisForJobQueue: RedisOptions & RedisOptionsSource;
 	redisForTimelines: RedisOptions & RedisOptionsSource;
 	redisForReactions: RedisOptions & RedisOptionsSource;
+	redisForRateLimit: RedisOptions & RedisOptionsSource;
 	sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
 	sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
 	perChannelMaxNoteCacheCount: number;
@@ -345,6 +347,7 @@ export function loadConfig(): Config {
 		redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
 		redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
 		redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
+		redisForRateLimit: config.redisForRateLimit ? convertRedisOptions(config.redisForRateLimit, host) : redis,
 		sentryForBackend: config.sentryForBackend,
 		sentryForFrontend: config.sentryForFrontend,
 		id: config.id,
@@ -535,7 +538,7 @@ function applyEnvOverrides(config: Source) {
 	_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]);
 	_apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]);
 	_apply_top([
-		['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions'],
+		['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'],
 		['host', 'port', 'username', 'pass', 'db', 'prefix'],
 	]);
 	_apply_top(['fulltextSearch', 'provider']);

packages/backend/src/di-symbols.ts

@@ -13,6 +13,7 @@ export const DI = {
 	redisForSub: Symbol('redisForSub'),
 	redisForTimelines: Symbol('redisForTimelines'),
 	redisForReactions: Symbol('redisForReactions'),
+	redisForRateLimit: Symbol('redisForRateLimit'),
 
 	//#region Repositories
 	usersRepository: Symbol('usersRepository'),

packages/backend/src/server/SkRateLimiterService.md

@@ -39,6 +39,7 @@ The first call is read-only, while the others perform at least one write operati
 Two integer keys are stored per client/subject, and both expire together after the maximum duration of the limit.
 While performance has not been formally tested, it's expected that SkRateLimiterService has an impact roughly on par with the legacy RateLimiterService.
 Redis memory usage should be notably lower due to the reduced number of keys and avoidance of set / array constructions.
+If redis load does become a concern, then a dedicated node can be assigned via the `redisForRateLimit` config setting.
 
 ## Concurrency and Multi-Node Correctness
 

packages/backend/src/server/SkRateLimiterService.ts

@@ -27,7 +27,7 @@ export class SkRateLimiterService {
 		@Inject('TimeService')
 		private readonly timeService: TimeService,
 
-		@Inject(DI.redis)
+		@Inject(DI.redisForRateLimit)
 		private readonly redisClient: Redis.Redis,
 
 		@Inject('RoleService')