diff --git a/ansible/deploy_task.yml b/ansible/deploy_task.yml index 23b219270..ab08408cd 100644 --- a/ansible/deploy_task.yml +++ b/ansible/deploy_task.yml @@ -19,8 +19,6 @@ when: item.service_name in inventory_hostname with_items: - service_name: task-celery - - service_name: task-beat - - service_name: task-search - service_name: task-work register: @@ -33,8 +31,6 @@ when: item.service_name in inventory_hostname with_items: - service_name: task-celery - - service_name: task-beat - - service_name: task-search - service_name: task-work tags: - never @@ -49,8 +45,6 @@ when: item.service_name in inventory_hostname with_items: - service_name: task-celery - - service_name: task-beat - - service_name: task-search - service_name: task-work tags: - never diff --git a/ansible/roles/celery_task/tasks/main.yml b/ansible/roles/celery_task/tasks/main.yml index b2870aaae..629afbbec 100644 --- a/ansible/roles/celery_task/tasks/main.yml +++ b/ansible/roles/celery_task/tasks/main.yml @@ -29,13 +29,9 @@ when: "item.container_name in inventory_hostname" with_items: - container_name: task-celery - command: "celery worker -A newsblur_web --loglevel=INFO -Q new_feeds,push_feeds,update_feeds" - - container_name: task-beat - command: "celery worker -A newsblur_web --loglevel=INFO -Q beat_feeds_task -c 1" - - container_name: task-search - command: "celery worker -A newsblur_web --loglevel=INFO -Q search_indexer -c 4" + command: "celery worker -A newsblur_web --loglevel=INFO -Q new_feeds,push_feeds,update_feeds,search_indexer" - container_name: task-work - command: "celery worker -A newsblur_web --loglevel=INFO -Q work_queue" + command: "celery worker -A newsblur_web --loglevel=INFO -Q beat_feeds_task,work_queue,cron_queue" - name: Register celery_task in consul tags: consul @@ -48,8 +44,6 @@ when: "item.service_name in inventory_hostname" with_items: - service_name: task-celery - - service_name: task-beat - - service_name: task-search - service_name: task-work - name: Reload celery @@ -59,7 +53,5 @@ when: app_changed.changed and item.container_name in inventory_hostname with_items: - container_name: task-celery - - container_name: task-beat - - container_name: task-search - container_name: task-work changed_when: app_changed.changed diff --git a/config/gunicorn_conf.py b/config/gunicorn_conf.py index f1fe499a0..5e1b75690 100644 --- a/config/gunicorn_conf.py +++ b/config/gunicorn_conf.py @@ -1,3 +1,4 @@ +import os import psutil import math @@ -29,3 +30,5 @@ if workers <= 4: if workers > 8: workers = 8 +if os.environ.get('DOCKERBUILD', False): + workers = 1 diff --git a/newsblur_web/docker_local_settings.py b/newsblur_web/docker_local_settings.py index 0655fedf8..0d87ace6c 100644 --- a/newsblur_web/docker_local_settings.py +++ b/newsblur_web/docker_local_settings.py @@ -102,6 +102,7 @@ MONGODB_SLAVE = { # Celery RabbitMQ/Redis Broker BROKER_URL = "redis://db_redis:6579/0" CELERY_RESULT_BACKEND = BROKER_URL +CELERY_WORKER_CONCURRENCY = 1 REDIS = { 'host': 'db_redis', diff --git a/terraform/digitalocean.tf b/terraform/digitalocean.tf index 59b35f150..ccc0d4093 100644 --- a/terraform/digitalocean.tf +++ b/terraform/digitalocean.tf @@ -391,7 +391,7 @@ resource "digitalocean_droplet" "db-mongo-analytics" { } resource "digitalocean_droplet" "task-celery" { - count = 2 + count = 10 image = var.droplet_os name = "task-celery${count.index+1}" region = var.droplet_region @@ -425,37 +425,3 @@ resource "digitalocean_droplet" "task-work" { command = "cd ..; ansible-playbook -l ${self.name} ansible/provision.yml" } } - -resource "digitalocean_droplet" "task-search" { - image = var.droplet_os - name = "task-search" - region = var.droplet_region - size = var.droplet_size - ssh_keys = [digitalocean_ssh_key.default.fingerprint] - provisioner "local-exec" { - command = "/srv/newsblur/ansible/utils/generate.py; sleep 120" - } - provisioner "local-exec" { - command = "cd ..; ansible-playbook -l ${self.name} ansible/setup_root.yml" - } - provisioner "local-exec" { - command = "cd ..; ansible-playbook -l ${self.name} ansible/provision.yml" - } -} - -resource "digitalocean_droplet" "task-beat" { - image = var.droplet_os - name = "task-beat" - region = var.droplet_region - size = var.droplet_size - ssh_keys = [digitalocean_ssh_key.default.fingerprint] - provisioner "local-exec" { - command = "/srv/newsblur/ansible/utils/generate.py; sleep 120" - } - provisioner "local-exec" { - command = "cd ..; ansible-playbook -l ${self.name} ansible/setup_root.yml" - } - provisioner "local-exec" { - command = "cd ..; ansible-playbook -l ${self.name} ansible/provision.yml" - } -} diff --git a/utils/feed_fetcher.py b/utils/feed_fetcher.py index 79fb3bda1..3d99b0153 100644 --- a/utils/feed_fetcher.py +++ b/utils/feed_fetcher.py @@ -957,7 +957,7 @@ class Dispatcher: self.feeds_count = feeds_count def run_jobs(self): - if self.options['single_threaded']: + if self.options['single_threaded'] or self.num_threads == 1: return dispatch_workers(self.feeds_queue[0], self.options) else: for i in range(self.num_threads):