public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-04-13 09:42:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Handling too many feeds tasked.

Browse source
This commit is contained in:
Samuel Clay 2024-02-28 09:33:10 -05:00
parent 25d6d84971
commit 1f5e9053dd
5 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ansible/roles/celery_task/tasks/main.yml
View file

@ -43,7 +43,7 @@
max-size: 100m max-size: 100m
healthcheck: healthcheck:
# test: celery inspect ping -A newsblur_web -d celery@$HOSTNAME # test: celery inspect ping -A newsblur_web -d celery@$HOSTNAME
test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 120)) && exit 0 || exit 1" test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 600)) && exit 0 || exit 1"
interval: 60s interval: 60s
timeout: 10s timeout: 10s
retries: 3 retries: 3

2
ansible/roles/postgres/tasks/main.yml
View file

@ -105,7 +105,7 @@
- /srv/newsblur/docker/postgres/postgres_ident-13.conf:/etc/postgresql/pg_ident.conf - /srv/newsblur/docker/postgres/postgres_ident-13.conf:/etc/postgresql/pg_ident.conf
- /home/nb/.ssh/id_rsa:/var/lib/postgresql/.ssh/id_rsa - /home/nb/.ssh/id_rsa:/var/lib/postgresql/.ssh/id_rsa
restart_policy: unless-stopped restart_policy: unless-stopped
when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres'] when: (inventory_hostname | regex_replace('\-?[0-9]+', '')) in ['db-postgres-primary', 'db-postgres', 'hdb-postgres', 'hdb-postgres-secondary']
# - name: Change ownership in postgres docker container # - name: Change ownership in postgres docker container
# become: yes # become: yes

1
ansible/roles/ufw/tasks/main.yml
View file

@ -6,6 +6,7 @@
do_hosts: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}" do_hosts: "{{ groups['NewsBlur_Docker'] | map('extract', hostvars, ['ansible_host']) }}"
- name: Generate UFW batch script - name: Generate UFW batch script
become: yes
template: template:
src: ufw_rules.sh.j2 src: ufw_rules.sh.j2
dest: /tmp/ufw_rules.sh dest: /tmp/ufw_rules.sh

8
apps/rss_feeds/tasks.py
View file

@ -2,13 +2,16 @@ import datetime
import os import os
import shutil import shutil
import time import time
import redis import redis
from newsblur_web.celeryapp import app
from celery.exceptions import SoftTimeLimitExceeded from celery.exceptions import SoftTimeLimitExceeded
from utils import log as logging
from django.conf import settings from django.conf import settings
from apps.profile.middleware import DBProfilerMiddleware from apps.profile.middleware import DBProfilerMiddleware
from newsblur_web.celeryapp import app
from utils import log as logging
from utils.redis_raw_log_middleware import RedisDumpMiddleware from utils.redis_raw_log_middleware import RedisDumpMiddleware
FEED_TASKING_MAX = 10000 FEED_TASKING_MAX = 10000
@app.task(name='task-feeds') @app.task(name='task-feeds')
@ -45,6 +48,7 @@ def TaskFeeds():
else: else:
logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size) logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
active_count = 0 active_count = 0
feeds = []
logging.debug(" ---> ~SN~FBTasking %s feeds took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % ( logging.debug(" ---> ~SN~FBTasking %s feeds took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
active_count, active_count,

4
docker/haproxy/haproxy.consul.cfg.j2
View file

@ -204,7 +204,7 @@ backend db_metrics
backend metrics backend metrics
option httpchk GET /_haproxychk option httpchk GET /_haproxychk
http-check expect rstatus 200|301 http-check expect rstatus 200|301
server app-django1 app-django1.node.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none server happ-web-01 happ-web-01.node.nyc1.consul:8000 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
backend postgres backend postgres
option httpchk GET /db_check/postgres option httpchk GET /db_check/postgres
@ -254,7 +254,7 @@ backend db_redis_sessions
backend db_redis_pubsub backend db_redis_pubsub
option httpchk GET /db_check/redis_pubsub option httpchk GET /db_check/redis_pubsub
default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
server hdb-redis-pubsub hdb-redis-pubsub.node.nyc1.consul:5579 server hdb-redis-pubsub db-redis-pubsub.service.nyc1.consul:5579
backend db_elasticsearch backend db_elasticsearch
option httpchk GET /db_check/elasticsearch option httpchk GET /db_check/elasticsearch