mirror of
https://github.com/viq/NewsBlur.git
synced 2025-11-01 09:09:16 +00:00
Adding task servers and updating automated deploy.
This commit is contained in:
Samuel Clay
parent
5354f56d01
commit
96cf1e2dec
3 changed files with 26 additions and 22 deletions
|
|
@ -8,12 +8,12 @@ set eventqueue
|
|||
|
||||
# If no feeds have been fetched in the last 10 minutes, something is wrong
|
||||
check file newsblur.log with path /home/sclay/newsblur/logs/newsblur.log
|
||||
if timestamp > 10 minutes then exec "/home/sclay/newsblur/utils/kill_celeryd.sh"
|
||||
if timestamp > 10 minutes then exec "/home/sclay/newsblur/utils/kill_celery.sh"
|
||||
as uid sclay and gid sclay
|
||||
|
||||
check system task_server
|
||||
if loadavg (1min) > 12 then exec "/home/sclay/newsblur/utils/kill_celeryd.sh"
|
||||
if loadavg (1min) > 12 then exec "/home/sclay/newsblur/utils/kill_celery.sh"
|
||||
as uid sclay and gid sclay
|
||||
if loadavg (5min) > 8 then exec "/home/sclay/newsblur/utils/kill_celeryd.sh"
|
||||
if loadavg (5min) > 8 then exec "/home/sclay/newsblur/utils/kill_celery.sh"
|
||||
as uid sclay and gid sclay
|
||||
|
||||
|
|
|
|||
35
fabfile.py
vendored
35
fabfile.py
vendored
|
|
@ -43,7 +43,10 @@ env.roledefs ={
|
|||
'task': ['task01.newsblur.com',
|
||||
'task02.newsblur.com',
|
||||
'task03.newsblur.com',
|
||||
'task04.newsblur.com'],
|
||||
'task04.newsblur.com',
|
||||
'task05.newsblur.com',
|
||||
'task06.newsblur.com',
|
||||
'task07.newsblur.com'],
|
||||
}
|
||||
|
||||
# ================
|
||||
|
|
@ -209,7 +212,8 @@ def setup_time_calibration():
|
|||
put('config/ntpdate.cron', env.NEWSBLUR_PATH)
|
||||
sudo('chmod 755 %s/ntpdate.cron' % env.NEWSBLUR_PATH)
|
||||
sudo('mv %s/ntpdate.cron /etc/cron.hourly/ntpdate' % env.NEWSBLUR_PATH)
|
||||
sudo('/etc/cron.hourly/ntpdate')
|
||||
with settings(warn_only=True):
|
||||
sudo('/etc/cron.hourly/ntpdate')
|
||||
|
||||
# =============
|
||||
# = Bootstrap =
|
||||
|
|
@ -228,8 +232,8 @@ def setup_common():
|
|||
setup_supervisor()
|
||||
setup_hosts()
|
||||
config_pgbouncer()
|
||||
setup_mongoengine()
|
||||
setup_forked_mongoengine()
|
||||
# setup_mongoengine()
|
||||
# setup_forked_mongoengine()
|
||||
setup_pymongo_repo()
|
||||
setup_logrotate()
|
||||
setup_nginx()
|
||||
|
|
@ -275,7 +279,7 @@ def setup_task():
|
|||
def setup_installs():
|
||||
sudo('apt-get -y update')
|
||||
sudo('apt-get -y upgrade')
|
||||
sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties libpcre3-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libmemcache0 python-memcache libyaml-0-2 python-yaml python-numpy python-scipy python-imaging munin munin-node munin-plugins-extra curl monit')
|
||||
sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libmemcache0 python-memcache libyaml-0-2 python-yaml python-numpy python-scipy python-imaging munin munin-node munin-plugins-extra curl monit')
|
||||
# sudo('add-apt-repository ppa:pitti/postgresql')
|
||||
sudo('apt-get -y update')
|
||||
sudo('apt-get -y install postgresql-client')
|
||||
|
|
@ -342,15 +346,15 @@ def setup_psycopg():
|
|||
sudo('easy_install -U psycopg2')
|
||||
|
||||
def setup_python():
|
||||
sudo('easy_install -U pip')
|
||||
sudo('easy_install -U fabric django readline pyflakes iconv celery django-celery django-celery-with-redis django-compress South django-extensions pymongo stripe BeautifulSoup pyyaml nltk==0.9.9 lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
|
||||
# sudo('easy_install -U pip')
|
||||
# sudo('easy_install -U fabric django==1.3.1 readline pyflakes iconv celery django-celery django-celery-with-redis django-compress South django-extensions pymongo stripe BeautifulSoup pyyaml nltk lxml oauth2 pytz boto seacucumber django_ses mongoengine redis requests')
|
||||
|
||||
put('config/pystartup.py', '.pystartup')
|
||||
with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):
|
||||
sudo('python setup.py install')
|
||||
# with cd(os.path.join(env.NEWSBLUR_PATH, 'vendor/cjson')):
|
||||
# sudo('python setup.py install')
|
||||
|
||||
with settings(warn_only=True):
|
||||
sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python/sitecustomize.py\'')
|
||||
sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'')
|
||||
|
||||
# PIL - Only if python-imaging didn't install through apt-get, like on Mac OS X.
|
||||
def setup_imaging():
|
||||
|
|
@ -381,7 +385,7 @@ def setup_mongoengine():
|
|||
with cd(env.VENDOR_PATH):
|
||||
with settings(warn_only=True):
|
||||
run('rm -fr mongoengine')
|
||||
run('git clone https://github.com/hmarr/mongoengine.git')
|
||||
run('git clone https://github.com/mongoengine/mongoengine.git')
|
||||
sudo('rm -f /usr/local/lib/python2.7/dist-packages/mongoengine')
|
||||
sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' %
|
||||
os.path.join(env.VENDOR_PATH, 'mongoengine/mongoengine'))
|
||||
|
|
@ -421,14 +425,15 @@ def setup_sudoers():
|
|||
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % env.user)
|
||||
|
||||
def setup_nginx():
|
||||
NGINX_VERSION = '1.2.0'
|
||||
with cd(env.VENDOR_PATH):
|
||||
with settings(warn_only=True):
|
||||
sudo("groupadd nginx")
|
||||
sudo("useradd -g nginx -d /var/www/htdocs -s /bin/false nginx")
|
||||
run('wget http://nginx.org/download/nginx-1.1.12.tar.gz')
|
||||
run('tar -xzf nginx-1.1.12.tar.gz')
|
||||
run('rm nginx-1.1.12.tar.gz')
|
||||
with cd('nginx-1.1.12'):
|
||||
run('wget http://nginx.org/download/nginx-%s.tar.gz' % NGINX_VERSION)
|
||||
run('tar -xzf nginx-%s.tar.gz' % NGINX_VERSION)
|
||||
run('rm nginx-%s.tar.gz' % NGINX_VERSION)
|
||||
with cd('nginx-%s' % NGINX_VERSION):
|
||||
run('./configure --with-http_ssl_module --with-http_stub_status_module --with-http_gzip_static_module')
|
||||
run('make')
|
||||
sudo('make install')
|
||||
|
|
|
|||
|
|
@ -76,9 +76,8 @@ class FetchFeed:
|
|||
etag=etag,
|
||||
modified=modified)
|
||||
|
||||
if self.options['verbose'] and getattr(self.fpf, 'status', None) == 200:
|
||||
logging.debug(u' ---> [%-30s] ~FBTIME: feed fetch in ~FM%.4ss' % (
|
||||
unicode(self.feed)[:30], time.time() - start))
|
||||
logging.debug(u' ---> [%-30s] ~FYFeed fetch in ~FM%.4ss' % (
|
||||
unicode(self.feed)[:30], time.time() - start))
|
||||
|
||||
return FEED_OK, self.fpf
|
||||
|
||||
|
|
@ -249,7 +248,7 @@ class ProcessFeed:
|
|||
elif link['rel'] == 'self':
|
||||
self_url = link['href']
|
||||
if hub_url and self_url and not settings.DEBUG:
|
||||
logging.debug(u' ---> [%-30s] ~BB~SK~FWSubscribing to PuSH hub: %s' % (
|
||||
logging.debug(u' ---> [%-30s] ~BB~FWSubscribing to PuSH hub: %s' % (
|
||||
unicode(self.feed)[:30], hub_url))
|
||||
PushSubscription.objects.subscribe(self_url, feed=self.feed, hub=hub_url)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue