public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-08-05 16:58:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into circular

Browse source 
* master:
  cleanup and fixing some of my stupid bugs
  cleanup and fixing some of my stupid bugs
  whoops, that section isn't called in the normal installation process. Reverted.
  Fix to make detecting if the install is a dev env work properly
  Fixed typo. Also, extracted most of the instances of a specific username out to `env.user`, so you don't need to search&replace to do an install. Lastly, added a separate function to change the shell of the host, so it's easier to disable.

Conflicts:
	fabfile.py
This commit is contained in:
Samuel Clay 2013-05-09 16:54:29 -07:00
commit 716d78525a
2 changed files with 125 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

221
fabfile.py vendored
View file

@ -33,11 +33,13 @@ env.NEWSBLUR_PATH = "~/projects/newsblur"
env.SECRETS_PATH = "~/projects/secrets-newsblur" env.SECRETS_PATH = "~/projects/secrets-newsblur"
env.VENDOR_PATH = "~/projects/code" env.VENDOR_PATH = "~/projects/code"
env.EC2_KEY_NAME = "sclay"
# ========= # =========
# = Roles = # = Roles =
# ========= # =========
env.user = 'sclay' env.user = 'sclay' # Why is this being set? It should be correctly set by fabric.
try: try:
hosts_path = os.path.expanduser(os.path.join(env.SECRETS_PATH, 'configs/hosts.yml')) hosts_path = os.path.expanduser(os.path.join(env.SECRETS_PATH, 'configs/hosts.yml'))
roles = yaml.load(open(hosts_path)) roles = yaml.load(open(hosts_path))
@ -69,7 +71,7 @@ def app():
def work(): def work():
server() server()
env.roles = ['work'] env.roles = ['work']
def dev(): def dev():
server() server()
env.roles = ['dev'] env.roles = ['dev']
@ -77,15 +79,15 @@ def dev():
def db(): def db():
server() server()
env.roles = ['db'] env.roles = ['db']
def task(): def task():
server() server()
env.roles = ['task'] env.roles = ['task']
def ec2task(): def ec2task():
ec2() ec2()
env.roles = ['ec2task'] env.roles = ['ec2task']
def vps(): def vps():
server() server()
env.roles = ['vps'] env.roles = ['vps']
@ -106,7 +108,7 @@ def ec2():
def all(): def all():
server() server()
env.roles = ['app', 'dev', 'db', 'task', 'debug'] env.roles = ['app', 'dev', 'db', 'task', 'debug']
# ========== # ==========
# = Deploy = # = Deploy =
# ========== # ==========
@ -122,7 +124,7 @@ def pre_deploy():
@serial @serial
def post_deploy(): def post_deploy():
cleanup_assets() cleanup_assets()
@parallel @parallel
def deploy(fast=False): def deploy(fast=False):
deploy_code(copy_assets=False, fast=fast) deploy_code(copy_assets=False, fast=fast)
@ -169,15 +171,15 @@ def deploy_node():
def gunicorn_restart(): def gunicorn_restart():
restart_gunicorn() restart_gunicorn()
def restart_gunicorn(): def restart_gunicorn():
with cd(env.NEWSBLUR_PATH), settings(warn_only=True): with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
run('sudo supervisorctl restart gunicorn') run('sudo supervisorctl restart gunicorn')
def gunicorn_stop(): def gunicorn_stop():
with cd(env.NEWSBLUR_PATH), settings(warn_only=True): with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
run('sudo supervisorctl stop gunicorn') run('sudo supervisorctl stop gunicorn')
def staging(): def staging():
with cd('~/staging'): with cd('~/staging'):
run('git pull') run('git pull')
@ -196,7 +198,7 @@ def staging_full():
@parallel @parallel
def celery(): def celery():
celery_slow() celery_slow()
def celery_slow(): def celery_slow():
with cd(env.NEWSBLUR_PATH): with cd(env.NEWSBLUR_PATH):
run('git pull') run('git pull')
@ -217,7 +219,7 @@ def celery_stop():
if env.user == 'ubuntu': if env.user == 'ubuntu':
sudo('./utils/kill_celery.sh') sudo('./utils/kill_celery.sh')
else: else:
run('./utils/kill_celery.sh') run('./utils/kill_celery.sh')
@parallel @parallel
def celery_start(): def celery_start():
@ -250,7 +252,7 @@ def transfer_assets():
def cleanup_assets(): def cleanup_assets():
local('rm -f static.tgz') local('rm -f static.tgz')
# =========== # ===========
# = Backups = # = Backups =
# =========== # ===========
@ -277,7 +279,7 @@ def sync_time():
sudo("/etc/init.d/ntp stop") sudo("/etc/init.d/ntp stop")
sudo("ntpdate pool.ntp.org") sudo("ntpdate pool.ntp.org")
sudo("/etc/init.d/ntp start") sudo("/etc/init.d/ntp start")
def setup_time_calibration(): def setup_time_calibration():
sudo('apt-get -y install ntp') sudo('apt-get -y install ntp')
put('config/ntpdate.cron', '%s/' % env.NEWSBLUR_PATH) put('config/ntpdate.cron', '%s/' % env.NEWSBLUR_PATH)
@ -286,13 +288,14 @@ def setup_time_calibration():
sudo('mv %s/ntpdate.cron /etc/cron.hourly/ntpdate' % env.NEWSBLUR_PATH) sudo('mv %s/ntpdate.cron /etc/cron.hourly/ntpdate' % env.NEWSBLUR_PATH)
with settings(warn_only=True): with settings(warn_only=True):
sudo('/etc/cron.hourly/ntpdate') sudo('/etc/cron.hourly/ntpdate')
# ============= # =============
# = Bootstrap = # = Bootstrap =
# ============= # =============
def setup_common(): def setup_common():
setup_installs() setup_installs()
change_shell()
setup_user() setup_user()
setup_sudoers() setup_sudoers()
setup_ulimit() setup_ulimit()
@ -317,7 +320,7 @@ def setup_all():
setup_app(skip_common=True) setup_app(skip_common=True)
setup_db(skip_common=True) setup_db(skip_common=True)
setup_task(skip_common=True) setup_task(skip_common=True)
def setup_app(skip_common=False): def setup_app(skip_common=False):
if not skip_common: if not skip_common:
setup_common() setup_common()
@ -355,7 +358,7 @@ def setup_db(engine=None, skip_common=False):
setup_redis(slave=True) setup_redis(slave=True)
setup_gunicorn(supervisor=False) setup_gunicorn(supervisor=False)
setup_db_munin() setup_db_munin()
# if env.user == 'ubuntu': # if env.user == 'ubuntu':
# setup_db_mdadm() # setup_db_mdadm()
@ -374,25 +377,29 @@ def setup_task(queue=None, skip_common=False):
# ================== # ==================
# = Setup - Common = # = Setup - Common =
# ================== # ==================
def setup_installs(): def setup_installs():
sudo('apt-get -y update') sudo('apt-get -y update')
sudo('apt-get -y upgrade') sudo('apt-get -y upgrade')
sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties software-properties-common libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libyaml-0-2 python-yaml python-numpy python-scipy python-imaging curl monit ufw') sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git python-dev locate python-software-properties software-properties-common libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libyaml-0-2 python-yaml python-numpy python-scipy python-imaging curl monit ufw')
# sudo('add-apt-repository ppa:pitti/postgresql') # sudo('add-apt-repository ppa:pitti/postgresql')
sudo('apt-get -y update') sudo('apt-get -y update')
sudo('apt-get -y install postgresql-client') sudo('apt-get -y install postgresql-client')
sudo('mkdir -p /var/run/postgresql') sudo('mkdir -p /var/run/postgresql')
sudo('chown postgres.postgres /var/run/postgresql') sudo('chown postgres.postgres /var/run/postgresql')
with settings(warn_only=True):
run('git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh')
run('curl -O http://peak.telecommunity.com/dist/ez_setup.py') run('curl -O http://peak.telecommunity.com/dist/ez_setup.py')
sudo('python ez_setup.py -U setuptools && rm ez_setup.py') sudo('python ez_setup.py -U setuptools && rm ez_setup.py')
sudo('chsh %s -s /bin/zsh' % env.user)
with settings(warn_only=True): with settings(warn_only=True):
sudo('mkdir -p %s' % env.VENDOR_PATH) sudo('mkdir -p %s' % env.VENDOR_PATH)
sudo('chown %s.%s %s' % (env.user, env.user, env.VENDOR_PATH)) sudo('chown %s.%s %s' % (env.user, env.user, env.VENDOR_PATH))
def change_shell():
sudo('apt-get -y install zsh')
with settings(warn_only=True):
run('git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh')
sudo('chsh %s -s /bin/zsh' % env.user)
def setup_user(): def setup_user():
# run('useradd -c "NewsBlur" -m newsblur -s /bin/zsh') # run('useradd -c "NewsBlur" -m newsblur -s /bin/zsh')
# run('openssl rand -base64 8 | tee -a ~conesus/.password | passwd -stdin conesus') # run('openssl rand -base64 8 | tee -a ~conesus/.password | passwd -stdin conesus')
@ -403,12 +410,12 @@ def setup_user():
put("~/.ssh/id_dsa.pub", "authorized_keys") put("~/.ssh/id_dsa.pub", "authorized_keys")
run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys') run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys')
run('rm authorized_keys') run('rm authorized_keys')
def add_machine_to_ssh(): def add_machine_to_ssh():
put("~/.ssh/id_dsa.pub", "local_keys") put("~/.ssh/id_dsa.pub", "local_keys")
run("echo `cat local_keys` >> .ssh/authorized_keys") run("echo `cat local_keys` >> .ssh/authorized_keys")
run("rm local_keys") run("rm local_keys")
def setup_repo(): def setup_repo():
with settings(warn_only=True): with settings(warn_only=True):
run('git clone https://github.com/samuelclay/NewsBlur.git ~/newsblur') run('git clone https://github.com/samuelclay/NewsBlur.git ~/newsblur')
@ -426,7 +433,7 @@ def setup_repo_local_settings():
def copy_local_settings(): def copy_local_settings():
with cd(env.NEWSBLUR_PATH): with cd(env.NEWSBLUR_PATH):
put('local_settings.py.server', 'local_settings.py') put('local_settings.py.server', 'local_settings.py')
def setup_local_files(): def setup_local_files():
put("config/toprc", "./.toprc") put("config/toprc", "./.toprc")
put("config/zshrc", "./.zshrc") put("config/zshrc", "./.zshrc")
@ -440,10 +447,10 @@ def setup_libxml_code():
with cd(env.VENDOR_PATH): with cd(env.VENDOR_PATH):
run('git clone git://git.gnome.org/libxml2') run('git clone git://git.gnome.org/libxml2')
run('git clone git://git.gnome.org/libxslt') run('git clone git://git.gnome.org/libxslt')
with cd(os.path.join(env.VENDOR_PATH, 'libxml2')): with cd(os.path.join(env.VENDOR_PATH, 'libxml2')):
run('./configure && make && sudo make install') run('./configure && make && sudo make install')
with cd(os.path.join(env.VENDOR_PATH, 'libxslt')): with cd(os.path.join(env.VENDOR_PATH, 'libxslt')):
run('./configure && make && sudo make install') run('./configure && make && sudo make install')
@ -461,7 +468,7 @@ def setup_python():
with settings(warn_only=True): with settings(warn_only=True):
sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'') sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'')
if env.user == 'ubuntu': if env.user == 'ubuntu':
with settings(warn_only=True): with settings(warn_only=True):
sudo('chown -R ubuntu.ubuntu /home/ubuntu/.python-eggs') sudo('chown -R ubuntu.ubuntu /home/ubuntu/.python-eggs')
@ -469,7 +476,7 @@ def setup_python():
# PIL - Only if python-imaging didn't install through apt-get, like on Mac OS X. # PIL - Only if python-imaging didn't install through apt-get, like on Mac OS X.
def setup_imaging(): def setup_imaging():
sudo('easy_install pil') sudo('easy_install pil')
def setup_supervisor(): def setup_supervisor():
sudo('apt-get -y install supervisor') sudo('apt-get -y install supervisor')
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True) put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True)
@ -500,31 +507,31 @@ def bounce_pgbouncer():
run('sleep 4') run('sleep 4')
run('sudo /etc/init.d/pgbouncer start', pty=False) run('sudo /etc/init.d/pgbouncer start', pty=False)
run('sleep 2') run('sleep 2')
def config_monit_task(): def config_monit_task():
put('config/monit_task.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True) put('config/monit_task.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True)
sudo('echo "startup=1" > /etc/default/monit') sudo('echo "startup=1" > /etc/default/monit')
sudo('/etc/init.d/monit restart') sudo('/etc/init.d/monit restart')
def config_monit_app(): def config_monit_app():
put('config/monit_app.conf', '/etc/monit/conf.d/gunicorn.conf', use_sudo=True) put('config/monit_app.conf', '/etc/monit/conf.d/gunicorn.conf', use_sudo=True)
sudo('echo "startup=1" > /etc/default/monit') sudo('echo "startup=1" > /etc/default/monit')
sudo('/etc/init.d/monit restart') sudo('/etc/init.d/monit restart')
def config_monit_db(): def config_monit_db():
put('config/monit_db.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True) put('config/monit_db.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True)
sudo('echo "startup=1" > /etc/default/monit') sudo('echo "startup=1" > /etc/default/monit')
sudo('/etc/init.d/monit restart') sudo('/etc/init.d/monit restart')
def setup_mongoengine(): def setup_mongoengine():
with cd(env.VENDOR_PATH), settings(warn_only=True): with cd(env.VENDOR_PATH), settings(warn_only=True):
run('rm -fr mongoengine') run('rm -fr mongoengine')
run('git clone https://github.com/MongoEngine/mongoengine.git') run('git clone https://github.com/MongoEngine/mongoengine.git')
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine')
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine-*') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine-*')
sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' % sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' %
os.path.join(env.VENDOR_PATH, 'mongoengine/mongoengine')) os.path.join(env.VENDOR_PATH, 'mongoengine/mongoengine'))
def setup_pymongo_repo(): def setup_pymongo_repo():
with cd(env.VENDOR_PATH), settings(warn_only=True): with cd(env.VENDOR_PATH), settings(warn_only=True):
run('git clone git://github.com/mongodb/mongo-python-driver.git pymongo') run('git clone git://github.com/mongodb/mongo-python-driver.git pymongo')
@ -532,10 +539,10 @@ def setup_pymongo_repo():
# sudo('python setup.py install') # sudo('python setup.py install')
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/pymongo*') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/pymongo*')
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/bson*') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/bson*')
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/gridgs*') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/gridfs*')
sudo('ln -fs %s /usr/local/lib/python2.7/dist-packages/' % sudo('ln -fs %s /usr/local/lib/python2.7/dist-packages/' %
os.path.join(env.VENDOR_PATH, 'pymongo/{pymongo,bson,gridfs}')) os.path.join(env.VENDOR_PATH, 'pymongo/{pymongo,bson,gridfs}'))
def setup_forked_mongoengine(): def setup_forked_mongoengine():
with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')), settings(warn_only=True): with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')), settings(warn_only=True):
run('git remote add clay https://github.com/samuelclay/mongoengine.git') run('git remote add clay https://github.com/samuelclay/mongoengine.git')
@ -551,7 +558,7 @@ def switch_forked_mongoengine():
# run('git checkout master') # run('git checkout master')
# run('get branch -D dev') # run('get branch -D dev')
# run('git checkout -b dev origin/dev') # run('git checkout -b dev origin/dev')
def setup_logrotate(): def setup_logrotate():
put('config/logrotate.conf', '/etc/logrotate.d/newsblur', use_sudo=True) put('config/logrotate.conf', '/etc/logrotate.d/newsblur', use_sudo=True)
put('config/logrotate.mongo.conf', '/etc/logrotate.d/mongodb', use_sudo=True) put('config/logrotate.mongo.conf', '/etc/logrotate.d/mongodb', use_sudo=True)
@ -571,7 +578,7 @@ def setup_ulimit():
run('echo "fs.file-max = 100000" >> /etc/sysctl.conf', pty=False) run('echo "fs.file-max = 100000" >> /etc/sysctl.conf', pty=False)
sudo('chmod 644 /etc/sysctl.conf', pty=False) sudo('chmod 644 /etc/sysctl.conf', pty=False)
sudo('sysctl -p') sudo('sysctl -p')
# run('touch /home/ubuntu/.bash_profile') # run('touch /home/ubuntu/.bash_profile')
# run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile') # run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile')
@ -579,7 +586,7 @@ def setup_ulimit():
# sudo chmod 666 /etc/sysctl.conf # sudo chmod 666 /etc/sysctl.conf
# echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf # echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf
# sudo chmod 644 /etc/sysctl.conf # sudo chmod 644 /etc/sysctl.conf
def setup_sudoers(user=None): def setup_sudoers(user=None):
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user)) sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
@ -596,7 +603,7 @@ def setup_nginx():
run('make') run('make')
sudo('make install') sudo('make install')
configure_nginx() configure_nginx()
def configure_nginx(): def configure_nginx():
put("config/nginx.conf", "/usr/local/nginx/conf/nginx.conf", use_sudo=True) put("config/nginx.conf", "/usr/local/nginx/conf/nginx.conf", use_sudo=True)
sudo("mkdir -p /usr/local/nginx/conf/sites-enabled") sudo("mkdir -p /usr/local/nginx/conf/sites-enabled")
@ -616,19 +623,19 @@ def setup_vps():
def setup_baremetal(): def setup_baremetal():
# Bare metal doesn't suffer from severe time drift. Use standard ntp slow-drift-calibration. # Bare metal doesn't suffer from severe time drift. Use standard ntp slow-drift-calibration.
sudo('apt-get -y install ntp') sudo('apt-get -y install ntp')
# =============== # ===============
# = Setup - App = # = Setup - App =
# =============== # ===============
def setup_app_firewall(): def setup_app_firewall():
sudo('ufw default deny') sudo('ufw default deny')
sudo('ufw allow ssh') # ssh sudo('ufw allow ssh') # ssh
sudo('ufw allow 80') # http sudo('ufw allow 80') # http
sudo('ufw allow 8000') # gunicorn sudo('ufw allow 8000') # gunicorn
sudo('ufw allow 8888') # socket.io sudo('ufw allow 8888') # socket.io
sudo('ufw allow 8889') # socket.io ssl sudo('ufw allow 8889') # socket.io ssl
sudo('ufw allow 443') # https sudo('ufw allow 443') # https
sudo('ufw --force enable') sudo('ufw --force enable')
def setup_app_motd(): def setup_app_motd():
@ -643,7 +650,7 @@ def setup_gunicorn(supervisor=True):
with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')): with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')):
run('git pull') run('git pull')
sudo('python setup.py develop') sudo('python setup.py develop')
def update_gunicorn(): def update_gunicorn():
with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')): with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')):
@ -696,8 +703,8 @@ def maintenance_off():
run('git checkout templates/maintenance_off.html') run('git checkout templates/maintenance_off.html')
def setup_haproxy(debug=False): def setup_haproxy(debug=False):
sudo('ufw allow 81') # nginx moved sudo('ufw allow 81') # nginx moved
sudo('ufw allow 1936') # haproxy stats sudo('ufw allow 1936') # haproxy stats
sudo('apt-get install -y haproxy') sudo('apt-get install -y haproxy')
sudo('apt-get remove -y haproxy') sudo('apt-get remove -y haproxy')
with cd(env.VENDOR_PATH): with cd(env.VENDOR_PATH):
@ -720,7 +727,7 @@ def setup_haproxy(debug=False):
run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path)) run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path))
put('config/haproxy_rsyslog.conf', '/etc/rsyslog.d/49-haproxy.conf', use_sudo=True) put('config/haproxy_rsyslog.conf', '/etc/rsyslog.d/49-haproxy.conf', use_sudo=True)
sudo('restart rsyslog') sudo('restart rsyslog')
sudo('/etc/init.d/haproxy stop') sudo('/etc/init.d/haproxy stop')
sudo('/etc/init.d/haproxy start') sudo('/etc/init.d/haproxy start')
@ -730,7 +737,7 @@ def config_haproxy(debug=False):
else: else:
put('../secrets-newsblur/configs/haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True) put('../secrets-newsblur/configs/haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True)
sudo('/etc/init.d/haproxy reload') sudo('/etc/init.d/haproxy reload')
def upgrade_django(): def upgrade_django():
with cd(env.NEWSBLUR_PATH), settings(warn_only=True): with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
sudo('supervisorctl stop gunicorn') sudo('supervisorctl stop gunicorn')
@ -752,10 +759,10 @@ def downgrade_pil():
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/Pillow*') sudo('rm -fr /usr/local/lib/python2.7/dist-packages/Pillow*')
pull() pull()
kill() kill()
# ============== # ==============
# = Setup - DB = # = Setup - DB =
# ============== # ==============
# @parallel # @parallel
def setup_db_firewall(): def setup_db_firewall():
@ -771,19 +778,19 @@ def setup_db_firewall():
sudo('ufw default deny') sudo('ufw default deny')
sudo('ufw allow ssh') sudo('ufw allow ssh')
sudo('ufw allow 80') sudo('ufw allow 80')
# DigitalOcean # DigitalOcean
for ip in set(env.roledefs['app'] + for ip in set(env.roledefs['app'] +
env.roledefs['db'] + env.roledefs['db'] +
env.roledefs['dev'] + env.roledefs['dev'] +
env.roledefs['debug'] + env.roledefs['debug'] +
env.roledefs['task'] + env.roledefs['task'] +
env.roledefs['node']): env.roledefs['node']):
sudo('ufw allow proto tcp from %s to any port %s' % ( sudo('ufw allow proto tcp from %s to any port %s' % (
ip, ip,
','.join(map(str, ports)) ','.join(map(str, ports))
)) ))
# EC2 # EC2
for host in set(env.roledefs['ec2task']): for host in set(env.roledefs['ec2task']):
ip = re.search('ec2-(\d+-\d+-\d+-\d+)', host).group(1).replace('-', '.') ip = re.search('ec2-(\d+-\d+-\d+-\d+)', host).group(1).replace('-', '.')
@ -793,10 +800,10 @@ def setup_db_firewall():
)) ))
sudo('ufw --force enable') sudo('ufw --force enable')
def setup_db_motd(): def setup_db_motd():
put('config/motd_db.txt', '/etc/motd.tail', use_sudo=True) put('config/motd_db.txt', '/etc/motd.tail', use_sudo=True)
def setup_rabbitmq(): def setup_rabbitmq():
sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list') sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list')
run('wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc') run('wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc')
@ -822,10 +829,10 @@ def setup_postgres(standby=False):
# sudo('echo "%s" > /proc/sys/kernel/shmmax' % shmmax) # sudo('echo "%s" > /proc/sys/kernel/shmmax' % shmmax)
# sudo('echo "\nkernel.shmmax = %s" > /etc/sysctl.conf' % shmmax) # sudo('echo "\nkernel.shmmax = %s" > /etc/sysctl.conf' % shmmax)
# sudo('sysctl -p') # sudo('sysctl -p')
if standby: if standby:
put('config/postgresql_recovery.conf', '/var/lib/postgresql/9.1/recovery.conf', use_sudo=True) put('config/postgresql_recovery.conf', '/var/lib/postgresql/9.1/recovery.conf', use_sudo=True)
sudo('/etc/init.d/postgresql stop') sudo('/etc/init.d/postgresql stop')
sudo('/etc/init.d/postgresql start') sudo('/etc/init.d/postgresql start')
@ -833,18 +840,18 @@ def copy_postgres_to_standby():
slave = 'db12.newsblur.com' slave = 'db12.newsblur.com'
# Make sure you can ssh from master to slave and back. # Make sure you can ssh from master to slave and back.
# Need to give postgres accounts keys in authroized_keys. # Need to give postgres accounts keys in authroized_keys.
# sudo('su postgres -c "psql -c \\"SELECT pg_start_backup(\'label\', true)\\""', pty=False) # sudo('su postgres -c "psql -c \\"SELECT pg_start_backup(\'label\', true)\\""', pty=False)
sudo('su postgres -c \"rsync -a --stats --progress /var/lib/postgresql/9.1/main postgres@%s:/var/lib/postgresql/9.1/ --exclude postmaster.pid\"' % slave, pty=False) sudo('su postgres -c \"rsync -a --stats --progress /var/lib/postgresql/9.1/main postgres@%s:/var/lib/postgresql/9.1/ --exclude postmaster.pid\"' % slave, pty=False)
# sudo('su postgres -c "psql -c \\"SELECT pg_stop_backup()\\""', pty=False) # sudo('su postgres -c "psql -c \\"SELECT pg_stop_backup()\\""', pty=False)
def setup_mongo(): def setup_mongo():
sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10') sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10')
# sudo('echo "deb http://downloads.mongodb.org/distros/ubuntu 10.10 10gen" >> /etc/apt/sources.list.d/10gen.list') # sudo('echo "deb http://downloads.mongodb.org/distros/ubuntu 10.10 10gen" >> /etc/apt/sources.list.d/10gen.list')
sudo('echo "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" >> /etc/apt/sources.list') sudo('echo "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" >> /etc/apt/sources.list')
sudo('apt-get update') sudo('apt-get update')
sudo('apt-get -y install mongodb-10gen') sudo('apt-get -y install mongodb-10gen')
put('config/mongodb.%s.conf' % ('prod' if env.user != 'ubuntu' else 'ec2'), put('config/mongodb.%s.conf' % ('prod' if env.user != 'ubuntu' else 'ec2'),
'/etc/mongodb.conf', use_sudo=True) '/etc/mongodb.conf', use_sudo=True)
run('echo "ulimit -n 10000" > mongodb.defaults') run('echo "ulimit -n 10000" > mongodb.defaults')
sudo('mv mongodb.defaults /etc/default/mongodb') sudo('mv mongodb.defaults /etc/default/mongodb')
@ -907,8 +914,8 @@ def setup_munin():
with settings(warn_only=True): with settings(warn_only=True):
sudo('/etc/init.d/spawn_fcgi_munin_graph start') sudo('/etc/init.d/spawn_fcgi_munin_graph start')
sudo('/etc/init.d/spawn_fcgi_munin_html start') sudo('/etc/init.d/spawn_fcgi_munin_html start')
def setup_db_munin(): def setup_db_munin():
sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH) sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
sudo('cp -frs %s/config/munin/pg_* /etc/munin/plugins/' % env.NEWSBLUR_PATH) sudo('cp -frs %s/config/munin/pg_* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
@ -927,7 +934,7 @@ def enable_celerybeat():
put('config/supervisor_celeryd_beat_feeds.conf', '/etc/supervisor/conf.d/celeryd_beat_feeds.conf', use_sudo=True) put('config/supervisor_celeryd_beat_feeds.conf', '/etc/supervisor/conf.d/celeryd_beat_feeds.conf', use_sudo=True)
sudo('supervisorctl reread') sudo('supervisorctl reread')
sudo('supervisorctl update') sudo('supervisorctl update')
def setup_db_mdadm(): def setup_db_mdadm():
sudo('apt-get -y install xfsprogs mdadm') sudo('apt-get -y install xfsprogs mdadm')
sudo('yes | mdadm --create /dev/md0 --level=0 -c256 --raid-devices=4 /dev/xvdf /dev/xvdg /dev/xvdh /dev/xvdi') sudo('yes | mdadm --create /dev/md0 --level=0 -c256 --raid-devices=4 /dev/xvdf /dev/xvdg /dev/xvdh /dev/xvdi')
@ -944,8 +951,8 @@ def setup_db_mdadm():
def setup_original_page_server(): def setup_original_page_server():
setup_node() setup_node()
sudo('mkdir -p /srv/originals') sudo('mkdir -p /srv/originals')
sudo('chown sclay.sclay -R /srv/originals') sudo('chown %s.%s -R /srv/originals' % (env.user, env.user)) # We assume that the group is the same name as the user. It's common on linux
put('config/supervisor_node_original.conf', put('config/supervisor_node_original.conf',
'/etc/supervisor/conf.d/node_original.conf', use_sudo=True) '/etc/supervisor/conf.d/node_original.conf', use_sudo=True)
sudo('supervisorctl reread') sudo('supervisorctl reread')
sudo('supervisorctl reload') sudo('supervisorctl reload')
@ -954,13 +961,13 @@ def setup_elasticsearch():
ES_VERSION = "0.90.0" ES_VERSION = "0.90.0"
sudo('apt-get update') sudo('apt-get update')
sudo('apt-get install openjdk-7-jre -y') sudo('apt-get install openjdk-7-jre -y')
with cd(env.VENDOR_PATH): with cd(env.VENDOR_PATH):
run('mkdir elasticsearch-%s' % ES_VERSION) run('mkdir elasticsearch-%s' % ES_VERSION)
with cd(os.path.join(env.VENDOR_PATH, 'elasticsearch-%s' % ES_VERSION)): with cd(os.path.join(env.VENDOR_PATH, 'elasticsearch-%s' % ES_VERSION)):
run('wget http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.deb' % ES_VERSION) run('wget http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.deb' % ES_VERSION)
sudo('dpkg -i elasticsearch-%s.deb' % ES_VERSION) sudo('dpkg -i elasticsearch-%s.deb' % ES_VERSION)
# ================ # ================
# = Setup - Task = # = Setup - Task =
# ================ # ================
@ -973,13 +980,13 @@ def setup_task_firewall():
def setup_task_motd(): def setup_task_motd():
put('config/motd_task.txt', '/etc/motd.tail', use_sudo=True) put('config/motd_task.txt', '/etc/motd.tail', use_sudo=True)
def enable_celery_supervisor(queue=None): def enable_celery_supervisor(queue=None):
if not queue: if not queue:
put('config/supervisor_celeryd.conf', '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True) put('config/supervisor_celeryd.conf', '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True)
else: else:
put('config/supervisor_celeryd_%s.conf' % queue, '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True) put('config/supervisor_celeryd_%s.conf' % queue, '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True)
sudo('supervisorctl reread') sudo('supervisorctl reread')
sudo('supervisorctl update') sudo('supervisorctl update')
@ -992,7 +999,7 @@ def copy_task_settings():
host = env.host.split('.', 2)[0] host = env.host.split('.', 2)[0]
else: else:
host = env.host_string.split('.', 2)[0] host = env.host_string.split('.', 2)[0]
with settings(warn_only=True): with settings(warn_only=True):
put('../secrets-newsblur/settings/task_settings.py', '%s/local_settings.py' % env.NEWSBLUR_PATH) put('../secrets-newsblur/settings/task_settings.py', '%s/local_settings.py' % env.NEWSBLUR_PATH)
run('echo "\nSERVER_NAME = \\\\"%s\\\\"" >> %s/local_settings.py' % (host, env.NEWSBLUR_PATH)) run('echo "\nSERVER_NAME = \\\\"%s\\\\"" >> %s/local_settings.py' % (host, env.NEWSBLUR_PATH))
@ -1011,14 +1018,14 @@ def setup_do(name, size=2):
region_id = doapi.regions()[0].id region_id = doapi.regions()[0].id
images = dict((s.name, s.id) for s in doapi.images()) images = dict((s.name, s.id) for s in doapi.images())
image_id = images[IMAGE_NAME] image_id = images[IMAGE_NAME]
instance = doapi.create_droplet(name=name, instance = doapi.create_droplet(name=name,
size_id=size_id, size_id=size_id,
image_id=image_id, image_id=image_id,
region_id=region_id, region_id=region_id,
ssh_key_ids=[str(ssh_key_id)], ssh_key_ids=[str(ssh_key_id)],
virtio=True) virtio=True)
print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE) print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE)
instance = doapi.show_droplet(instance.id) instance = doapi.show_droplet(instance.id)
i = 0 i = 0
while True: while True:
@ -1035,38 +1042,39 @@ def setup_do(name, size=2):
else: else:
print "!!! Error: %s" % instance.status print "!!! Error: %s" % instance.status
return return
host = instance.ip_address host = instance.ip_address
env.host_string = host env.host_string = host
time.sleep(10) time.sleep(10)
add_user_to_do() add_user_to_do()
def add_user_to_do(): def add_user_to_do():
env.user = "root" env.user = "root"
repo_user = "sclay"
with settings(warn_only=True): with settings(warn_only=True):
run('useradd -m sclay') run('useradd -m %s' % (repo_user))
setup_sudoers("sclay") setup_sudoers("%s" % (repo_user))
run('mkdir -p ~sclay/.ssh && chmod 700 ~sclay/.ssh') run('mkdir -p ~%s/.ssh && chmod 700 ~%s/.ssh' % (repo_user, repo_user))
run('rm -fr ~sclay/.ssh/id_dsa*') run('rm -fr ~%s/.ssh/id_dsa*' % (repo_user))
run('ssh-keygen -t dsa -f ~sclay/.ssh/id_dsa -N ""') run('ssh-keygen -t dsa -f ~%s/.ssh/id_dsa -N ""' % (repo_user, repo_user))
run('touch ~sclay/.ssh/authorized_keys') run('touch ~%s/.ssh/authorized_keys' % (repo_user, repo_user))
put("~/.ssh/id_dsa.pub", "authorized_keys") put("~/.ssh/id_dsa.pub", "authorized_keys")
run('echo `cat authorized_keys` >> ~sclay/.ssh/authorized_keys') run('echo `cat authorized_keys` >> ~%s/.ssh/authorized_keys' % (repo_user))
run('rm authorized_keys') run('rm authorized_keys')
run('chown sclay.sclay -R ~sclay/.ssh') run('chown %s.%s -R ~%s/.ssh' % (repo_user, repo_user, repo_user))
env.user = "sclay" env.user = repo_user
# =============== # ===============
# = Setup - EC2 = # = Setup - EC2 =
# =============== # ===============
def setup_ec2(): def setup_ec2():
AMI_NAME = 'ami-834cf1ea' # Ubuntu 64-bit 12.04 LTS AMI_NAME = 'ami-834cf1ea' # Ubuntu 64-bit 12.04 LTS
# INSTANCE_TYPE = 'c1.medium' # INSTANCE_TYPE = 'c1.medium'
INSTANCE_TYPE = 'c1.medium' INSTANCE_TYPE = 'c1.medium'
conn = EC2Connection(django_settings.AWS_ACCESS_KEY_ID, django_settings.AWS_SECRET_ACCESS_KEY) conn = EC2Connection(django_settings.AWS_ACCESS_KEY_ID, django_settings.AWS_SECRET_ACCESS_KEY)
reservation = conn.run_instances(AMI_NAME, instance_type=INSTANCE_TYPE, reservation = conn.run_instances(AMI_NAME, instance_type=INSTANCE_TYPE,
key_name='sclay', key_name=env.EC2_KEY_NAME,
security_groups=['db-mongo']) security_groups=['db-mongo'])
instance = reservation.instances[0] instance = reservation.instances[0]
print "Booting reservation: %s/%s (size: %s)" % (reservation, instance, INSTANCE_TYPE) print "Booting reservation: %s/%s (size: %s)" % (reservation, instance, INSTANCE_TYPE)
@ -1085,12 +1093,12 @@ def setup_ec2():
else: else:
print "!!! Error: %s" % instance.state print "!!! Error: %s" % instance.state
return return
host = instance.public_dns_name host = instance.public_dns_name
env.host_string = host env.host_string = host
# ============== # ==============
# = Tasks - DB = # = Tasks - DB =
# ============== # ==============
@ -1098,19 +1106,20 @@ def setup_ec2():
def restore_postgres(port=5433): def restore_postgres(port=5433):
backup_date = '2013-01-29-09-00' backup_date = '2013-01-29-09-00'
yes = prompt("Dropping and creating NewsBlur PGSQL db. Sure?") yes = prompt("Dropping and creating NewsBlur PGSQL db. Sure?")
if yes != 'y': return if yes != 'y':
return
# run('PYTHONPATH=%s python utils/backups/s3.py get backup_postgresql_%s.sql.gz' % (env.NEWSBLUR_PATH, backup_date)) # run('PYTHONPATH=%s python utils/backups/s3.py get backup_postgresql_%s.sql.gz' % (env.NEWSBLUR_PATH, backup_date))
# sudo('su postgres -c "createuser -p %s -U newsblur"' % (port,)) # sudo('su postgres -c "createuser -p %s -U newsblur"' % (port,))
run('dropdb newsblur -p %s -U postgres' % (port,), pty=False) run('dropdb newsblur -p %s -U postgres' % (port,), pty=False)
run('createdb newsblur -p %s -O newsblur' % (port,), pty=False) run('createdb newsblur -p %s -O newsblur' % (port,), pty=False)
run('pg_restore -p %s --role=newsblur --dbname=newsblur /Users/sclay/Documents/backups/backup_postgresql_%s.sql.gz' % (port, backup_date), pty=False) run('pg_restore -p %s --role=newsblur --dbname=newsblur /Users/sclay/Documents/backups/backup_postgresql_%s.sql.gz' % (port, backup_date), pty=False)
def restore_mongo(): def restore_mongo():
backup_date = '2012-07-24-09-00' backup_date = '2012-07-24-09-00'
run('PYTHONPATH=/home/%s/newsblur python s3.py get backup_mongo_%s.tgz' % (env.user, backup_date)) run('PYTHONPATH=/home/%s/newsblur python s3.py get backup_mongo_%s.tgz' % (env.user, backup_date))
run('tar -xf backup_mongo_%s.tgz' % backup_date) run('tar -xf backup_mongo_%s.tgz' % backup_date)
run('mongorestore backup_mongo_%s' % backup_date) run('mongorestore backup_mongo_%s' % backup_date)
# ====== # ======
# = S3 = # = S3 =
# ====== # ======

20
settings.py
View file

@ -80,7 +80,7 @@ ALLOWED_HOSTS = ['*']
PRODUCTION = NEWSBLUR_DIR.find('/home/conesus/newsblur') == 0 PRODUCTION = NEWSBLUR_DIR.find('/home/conesus/newsblur') == 0
STAGING = NEWSBLUR_DIR.find('/home/conesus/staging') == 0 STAGING = NEWSBLUR_DIR.find('/home/conesus/staging') == 0
DEVELOPMENT = NEWSBLUR_DIR.find('/Users/') == 0 DEVELOPMENT = (NEWSBLUR_DIR.find('/Users/') == 0)
# =========================== # ===========================
# = Django-specific Modules = # = Django-specific Modules =
@ -118,7 +118,7 @@ LOGGING = {
'disable_existing_loggers': False, 'disable_existing_loggers': False,
'formatters': { 'formatters': {
'verbose': { 'verbose': {
'format': '[%(asctime)-12s] %(message)s', 'format': '[%(asctime)-12s] %(message)s',
'datefmt': '%b %d %H:%M:%S' 'datefmt': '%b %d %H:%M:%S'
}, },
'simple': { 'simple': {
@ -192,7 +192,7 @@ ROOT_URLCONF = 'urls'
INTERNAL_IPS = ('127.0.0.1',) INTERNAL_IPS = ('127.0.0.1',)
LOGGING_LOG_SQL = True LOGGING_LOG_SQL = True
APPEND_SLASH = False APPEND_SLASH = False
SOUTH_TESTS_MIGRATE = False SOUTH_TESTS_MIGRATE = False
SESSION_ENGINE = "django.contrib.sessions.backends.db" SESSION_ENGINE = "django.contrib.sessions.backends.db"
TEST_RUNNER = "utils.testrunner.TestRunner" TEST_RUNNER = "utils.testrunner.TestRunner"
SESSION_COOKIE_NAME = 'newsblur_sessionid' SESSION_COOKIE_NAME = 'newsblur_sessionid'
@ -323,8 +323,8 @@ CELERY_QUEUES = {
CELERY_DEFAULT_QUEUE = "work_queue" CELERY_DEFAULT_QUEUE = "work_queue"
CELERYD_PREFETCH_MULTIPLIER = 1 CELERYD_PREFETCH_MULTIPLIER = 1
CELERY_IMPORTS = ("apps.rss_feeds.tasks", CELERY_IMPORTS = ("apps.rss_feeds.tasks",
"apps.social.tasks", "apps.social.tasks",
"apps.reader.tasks", "apps.reader.tasks",
"apps.feed_import.tasks", "apps.feed_import.tasks",
"apps.statistics.tasks",) "apps.statistics.tasks",)
@ -398,22 +398,22 @@ MONGO_ANALYTICS_DB = {
class MasterSlaveRouter(object): class MasterSlaveRouter(object):
"""A router that sets up a simple master/slave configuration""" """A router that sets up a simple master/slave configuration"""
def db_for_read(self, model, **hints): def db_for_read(self, model, **hints):
"Point all read operations to a random slave" "Point all read operations to a random slave"
return 'slave' return 'slave'
def db_for_write(self, model, **hints): def db_for_write(self, model, **hints):
"Point all write operations to the master" "Point all write operations to the master"
return 'default' return 'default'
def allow_relation(self, obj1, obj2, **hints): def allow_relation(self, obj1, obj2, **hints):
"Allow any relation between two objects in the db pool" "Allow any relation between two objects in the db pool"
db_list = ('slave','default') db_list = ('slave','default')
if obj1._state.db in db_list and obj2._state.db in db_list: if obj1._state.db in db_list and obj2._state.db in db_list:
return True return True
return None return None
def allow_syncdb(self, db, model): def allow_syncdb(self, db, model):
"Explicitly put all models on all databases." "Explicitly put all models on all databases."
return True return True
@ -474,7 +474,7 @@ if not DEVELOPMENT:
INSTALLED_APPS += ( INSTALLED_APPS += (
'gunicorn', 'gunicorn',
'raven.contrib.django', 'raven.contrib.django',
'django_ses', 'django_ses',
) )