mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-08-05 16:58:59 +00:00
Merge branch 'master' into circular
* master: cleanup and fixing some of my stupid bugs cleanup and fixing some of my stupid bugs whoops, that section isn't called in the normal installation process. Reverted. Fix to make detecting if the install is a dev env work properly Fixed typo. Also, extracted most of the instances of a specific username out to `env.user`, so you don't need to search&replace to do an install. Lastly, added a separate function to change the shell of the host, so it's easier to disable. Conflicts: fabfile.py
This commit is contained in:
Samuel Clay
commit
716d78525a
2 changed files with 125 additions and 116 deletions
221
fabfile.py
vendored
221
fabfile.py
vendored
|
|
@ -33,11 +33,13 @@ env.NEWSBLUR_PATH = "~/projects/newsblur"
|
|||
env.SECRETS_PATH = "~/projects/secrets-newsblur"
|
||||
env.VENDOR_PATH = "~/projects/code"
|
||||
|
||||
env.EC2_KEY_NAME = "sclay"
|
||||
|
||||
# =========
|
||||
# = Roles =
|
||||
# =========
|
||||
|
||||
env.user = 'sclay'
|
||||
env.user = 'sclay' # Why is this being set? It should be correctly set by fabric.
|
||||
try:
|
||||
hosts_path = os.path.expanduser(os.path.join(env.SECRETS_PATH, 'configs/hosts.yml'))
|
||||
roles = yaml.load(open(hosts_path))
|
||||
|
|
@ -69,7 +71,7 @@ def app():
|
|||
def work():
|
||||
server()
|
||||
env.roles = ['work']
|
||||
|
||||
|
||||
def dev():
|
||||
server()
|
||||
env.roles = ['dev']
|
||||
|
|
@ -77,15 +79,15 @@ def dev():
|
|||
def db():
|
||||
server()
|
||||
env.roles = ['db']
|
||||
|
||||
|
||||
def task():
|
||||
server()
|
||||
env.roles = ['task']
|
||||
|
||||
|
||||
def ec2task():
|
||||
ec2()
|
||||
env.roles = ['ec2task']
|
||||
|
||||
|
||||
def vps():
|
||||
server()
|
||||
env.roles = ['vps']
|
||||
|
|
@ -106,7 +108,7 @@ def ec2():
|
|||
def all():
|
||||
server()
|
||||
env.roles = ['app', 'dev', 'db', 'task', 'debug']
|
||||
|
||||
|
||||
# ==========
|
||||
# = Deploy =
|
||||
# ==========
|
||||
|
|
@ -122,7 +124,7 @@ def pre_deploy():
|
|||
@serial
|
||||
def post_deploy():
|
||||
cleanup_assets()
|
||||
|
||||
|
||||
@parallel
|
||||
def deploy(fast=False):
|
||||
deploy_code(copy_assets=False, fast=fast)
|
||||
|
|
@ -169,15 +171,15 @@ def deploy_node():
|
|||
|
||||
def gunicorn_restart():
|
||||
restart_gunicorn()
|
||||
|
||||
|
||||
def restart_gunicorn():
|
||||
with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
|
||||
run('sudo supervisorctl restart gunicorn')
|
||||
|
||||
|
||||
def gunicorn_stop():
|
||||
with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
|
||||
run('sudo supervisorctl stop gunicorn')
|
||||
|
||||
|
||||
def staging():
|
||||
with cd('~/staging'):
|
||||
run('git pull')
|
||||
|
|
@ -196,7 +198,7 @@ def staging_full():
|
|||
@parallel
|
||||
def celery():
|
||||
celery_slow()
|
||||
|
||||
|
||||
def celery_slow():
|
||||
with cd(env.NEWSBLUR_PATH):
|
||||
run('git pull')
|
||||
|
|
@ -217,7 +219,7 @@ def celery_stop():
|
|||
if env.user == 'ubuntu':
|
||||
sudo('./utils/kill_celery.sh')
|
||||
else:
|
||||
run('./utils/kill_celery.sh')
|
||||
run('./utils/kill_celery.sh')
|
||||
|
||||
@parallel
|
||||
def celery_start():
|
||||
|
|
@ -250,7 +252,7 @@ def transfer_assets():
|
|||
|
||||
def cleanup_assets():
|
||||
local('rm -f static.tgz')
|
||||
|
||||
|
||||
# ===========
|
||||
# = Backups =
|
||||
# ===========
|
||||
|
|
@ -277,7 +279,7 @@ def sync_time():
|
|||
sudo("/etc/init.d/ntp stop")
|
||||
sudo("ntpdate pool.ntp.org")
|
||||
sudo("/etc/init.d/ntp start")
|
||||
|
||||
|
||||
def setup_time_calibration():
|
||||
sudo('apt-get -y install ntp')
|
||||
put('config/ntpdate.cron', '%s/' % env.NEWSBLUR_PATH)
|
||||
|
|
@ -286,13 +288,14 @@ def setup_time_calibration():
|
|||
sudo('mv %s/ntpdate.cron /etc/cron.hourly/ntpdate' % env.NEWSBLUR_PATH)
|
||||
with settings(warn_only=True):
|
||||
sudo('/etc/cron.hourly/ntpdate')
|
||||
|
||||
|
||||
# =============
|
||||
# = Bootstrap =
|
||||
# =============
|
||||
|
||||
def setup_common():
|
||||
setup_installs()
|
||||
change_shell()
|
||||
setup_user()
|
||||
setup_sudoers()
|
||||
setup_ulimit()
|
||||
|
|
@ -317,7 +320,7 @@ def setup_all():
|
|||
setup_app(skip_common=True)
|
||||
setup_db(skip_common=True)
|
||||
setup_task(skip_common=True)
|
||||
|
||||
|
||||
def setup_app(skip_common=False):
|
||||
if not skip_common:
|
||||
setup_common()
|
||||
|
|
@ -355,7 +358,7 @@ def setup_db(engine=None, skip_common=False):
|
|||
setup_redis(slave=True)
|
||||
setup_gunicorn(supervisor=False)
|
||||
setup_db_munin()
|
||||
|
||||
|
||||
# if env.user == 'ubuntu':
|
||||
# setup_db_mdadm()
|
||||
|
||||
|
|
@ -374,25 +377,29 @@ def setup_task(queue=None, skip_common=False):
|
|||
# ==================
|
||||
# = Setup - Common =
|
||||
# ==================
|
||||
|
||||
|
||||
def setup_installs():
|
||||
sudo('apt-get -y update')
|
||||
sudo('apt-get -y upgrade')
|
||||
sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git zsh python-dev locate python-software-properties software-properties-common libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libyaml-0-2 python-yaml python-numpy python-scipy python-imaging curl monit ufw')
|
||||
sudo('apt-get -y install build-essential gcc scons libreadline-dev sysstat iotop git python-dev locate python-software-properties software-properties-common libpcre3-dev libncurses5-dev libdbd-pg-perl libssl-dev make pgbouncer python-psycopg2 libyaml-0-2 python-yaml python-numpy python-scipy python-imaging curl monit ufw')
|
||||
|
||||
# sudo('add-apt-repository ppa:pitti/postgresql')
|
||||
sudo('apt-get -y update')
|
||||
sudo('apt-get -y install postgresql-client')
|
||||
sudo('mkdir -p /var/run/postgresql')
|
||||
sudo('chown postgres.postgres /var/run/postgresql')
|
||||
with settings(warn_only=True):
|
||||
run('git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh')
|
||||
run('curl -O http://peak.telecommunity.com/dist/ez_setup.py')
|
||||
sudo('python ez_setup.py -U setuptools && rm ez_setup.py')
|
||||
sudo('chsh %s -s /bin/zsh' % env.user)
|
||||
with settings(warn_only=True):
|
||||
sudo('mkdir -p %s' % env.VENDOR_PATH)
|
||||
sudo('chown %s.%s %s' % (env.user, env.user, env.VENDOR_PATH))
|
||||
|
||||
|
||||
def change_shell():
|
||||
sudo('apt-get -y install zsh')
|
||||
with settings(warn_only=True):
|
||||
run('git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh')
|
||||
sudo('chsh %s -s /bin/zsh' % env.user)
|
||||
|
||||
def setup_user():
|
||||
# run('useradd -c "NewsBlur" -m newsblur -s /bin/zsh')
|
||||
# run('openssl rand -base64 8 | tee -a ~conesus/.password | passwd -stdin conesus')
|
||||
|
|
@ -403,12 +410,12 @@ def setup_user():
|
|||
put("~/.ssh/id_dsa.pub", "authorized_keys")
|
||||
run('echo `cat authorized_keys` >> ~/.ssh/authorized_keys')
|
||||
run('rm authorized_keys')
|
||||
|
||||
|
||||
def add_machine_to_ssh():
|
||||
put("~/.ssh/id_dsa.pub", "local_keys")
|
||||
run("echo `cat local_keys` >> .ssh/authorized_keys")
|
||||
run("rm local_keys")
|
||||
|
||||
|
||||
def setup_repo():
|
||||
with settings(warn_only=True):
|
||||
run('git clone https://github.com/samuelclay/NewsBlur.git ~/newsblur')
|
||||
|
|
@ -426,7 +433,7 @@ def setup_repo_local_settings():
|
|||
def copy_local_settings():
|
||||
with cd(env.NEWSBLUR_PATH):
|
||||
put('local_settings.py.server', 'local_settings.py')
|
||||
|
||||
|
||||
def setup_local_files():
|
||||
put("config/toprc", "./.toprc")
|
||||
put("config/zshrc", "./.zshrc")
|
||||
|
|
@ -440,10 +447,10 @@ def setup_libxml_code():
|
|||
with cd(env.VENDOR_PATH):
|
||||
run('git clone git://git.gnome.org/libxml2')
|
||||
run('git clone git://git.gnome.org/libxslt')
|
||||
|
||||
|
||||
with cd(os.path.join(env.VENDOR_PATH, 'libxml2')):
|
||||
run('./configure && make && sudo make install')
|
||||
|
||||
|
||||
with cd(os.path.join(env.VENDOR_PATH, 'libxslt')):
|
||||
run('./configure && make && sudo make install')
|
||||
|
||||
|
|
@ -461,7 +468,7 @@ def setup_python():
|
|||
|
||||
with settings(warn_only=True):
|
||||
sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'')
|
||||
|
||||
|
||||
if env.user == 'ubuntu':
|
||||
with settings(warn_only=True):
|
||||
sudo('chown -R ubuntu.ubuntu /home/ubuntu/.python-eggs')
|
||||
|
|
@ -469,7 +476,7 @@ def setup_python():
|
|||
# PIL - Only if python-imaging didn't install through apt-get, like on Mac OS X.
|
||||
def setup_imaging():
|
||||
sudo('easy_install pil')
|
||||
|
||||
|
||||
def setup_supervisor():
|
||||
sudo('apt-get -y install supervisor')
|
||||
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True)
|
||||
|
|
@ -500,31 +507,31 @@ def bounce_pgbouncer():
|
|||
run('sleep 4')
|
||||
run('sudo /etc/init.d/pgbouncer start', pty=False)
|
||||
run('sleep 2')
|
||||
|
||||
|
||||
def config_monit_task():
|
||||
put('config/monit_task.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True)
|
||||
sudo('echo "startup=1" > /etc/default/monit')
|
||||
sudo('/etc/init.d/monit restart')
|
||||
|
||||
|
||||
def config_monit_app():
|
||||
put('config/monit_app.conf', '/etc/monit/conf.d/gunicorn.conf', use_sudo=True)
|
||||
sudo('echo "startup=1" > /etc/default/monit')
|
||||
sudo('/etc/init.d/monit restart')
|
||||
|
||||
|
||||
def config_monit_db():
|
||||
put('config/monit_db.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True)
|
||||
sudo('echo "startup=1" > /etc/default/monit')
|
||||
sudo('/etc/init.d/monit restart')
|
||||
|
||||
|
||||
def setup_mongoengine():
|
||||
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
||||
run('rm -fr mongoengine')
|
||||
run('git clone https://github.com/MongoEngine/mongoengine.git')
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine')
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine-*')
|
||||
sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' %
|
||||
sudo('ln -s %s /usr/local/lib/python2.7/dist-packages/mongoengine' %
|
||||
os.path.join(env.VENDOR_PATH, 'mongoengine/mongoengine'))
|
||||
|
||||
|
||||
def setup_pymongo_repo():
|
||||
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
||||
run('git clone git://github.com/mongodb/mongo-python-driver.git pymongo')
|
||||
|
|
@ -532,10 +539,10 @@ def setup_pymongo_repo():
|
|||
# sudo('python setup.py install')
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/pymongo*')
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/bson*')
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/gridgs*')
|
||||
sudo('ln -fs %s /usr/local/lib/python2.7/dist-packages/' %
|
||||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/gridfs*')
|
||||
sudo('ln -fs %s /usr/local/lib/python2.7/dist-packages/' %
|
||||
os.path.join(env.VENDOR_PATH, 'pymongo/{pymongo,bson,gridfs}'))
|
||||
|
||||
|
||||
def setup_forked_mongoengine():
|
||||
with cd(os.path.join(env.VENDOR_PATH, 'mongoengine')), settings(warn_only=True):
|
||||
run('git remote add clay https://github.com/samuelclay/mongoengine.git')
|
||||
|
|
@ -551,7 +558,7 @@ def switch_forked_mongoengine():
|
|||
# run('git checkout master')
|
||||
# run('get branch -D dev')
|
||||
# run('git checkout -b dev origin/dev')
|
||||
|
||||
|
||||
def setup_logrotate():
|
||||
put('config/logrotate.conf', '/etc/logrotate.d/newsblur', use_sudo=True)
|
||||
put('config/logrotate.mongo.conf', '/etc/logrotate.d/mongodb', use_sudo=True)
|
||||
|
|
@ -571,7 +578,7 @@ def setup_ulimit():
|
|||
run('echo "fs.file-max = 100000" >> /etc/sysctl.conf', pty=False)
|
||||
sudo('chmod 644 /etc/sysctl.conf', pty=False)
|
||||
sudo('sysctl -p')
|
||||
|
||||
|
||||
# run('touch /home/ubuntu/.bash_profile')
|
||||
# run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile')
|
||||
|
||||
|
|
@ -579,7 +586,7 @@ def setup_ulimit():
|
|||
# sudo chmod 666 /etc/sysctl.conf
|
||||
# echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf
|
||||
# sudo chmod 644 /etc/sysctl.conf
|
||||
|
||||
|
||||
def setup_sudoers(user=None):
|
||||
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
|
||||
|
||||
|
|
@ -596,7 +603,7 @@ def setup_nginx():
|
|||
run('make')
|
||||
sudo('make install')
|
||||
configure_nginx()
|
||||
|
||||
|
||||
def configure_nginx():
|
||||
put("config/nginx.conf", "/usr/local/nginx/conf/nginx.conf", use_sudo=True)
|
||||
sudo("mkdir -p /usr/local/nginx/conf/sites-enabled")
|
||||
|
|
@ -616,19 +623,19 @@ def setup_vps():
|
|||
def setup_baremetal():
|
||||
# Bare metal doesn't suffer from severe time drift. Use standard ntp slow-drift-calibration.
|
||||
sudo('apt-get -y install ntp')
|
||||
|
||||
|
||||
# ===============
|
||||
# = Setup - App =
|
||||
# ===============
|
||||
|
||||
def setup_app_firewall():
|
||||
sudo('ufw default deny')
|
||||
sudo('ufw allow ssh') # ssh
|
||||
sudo('ufw allow 80') # http
|
||||
sudo('ufw allow 8000') # gunicorn
|
||||
sudo('ufw allow 8888') # socket.io
|
||||
sudo('ufw allow 8889') # socket.io ssl
|
||||
sudo('ufw allow 443') # https
|
||||
sudo('ufw allow ssh') # ssh
|
||||
sudo('ufw allow 80') # http
|
||||
sudo('ufw allow 8000') # gunicorn
|
||||
sudo('ufw allow 8888') # socket.io
|
||||
sudo('ufw allow 8889') # socket.io ssl
|
||||
sudo('ufw allow 443') # https
|
||||
sudo('ufw --force enable')
|
||||
|
||||
def setup_app_motd():
|
||||
|
|
@ -643,7 +650,7 @@ def setup_gunicorn(supervisor=True):
|
|||
with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')):
|
||||
run('git pull')
|
||||
sudo('python setup.py develop')
|
||||
|
||||
|
||||
|
||||
def update_gunicorn():
|
||||
with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')):
|
||||
|
|
@ -696,8 +703,8 @@ def maintenance_off():
|
|||
run('git checkout templates/maintenance_off.html')
|
||||
|
||||
def setup_haproxy(debug=False):
|
||||
sudo('ufw allow 81') # nginx moved
|
||||
sudo('ufw allow 1936') # haproxy stats
|
||||
sudo('ufw allow 81') # nginx moved
|
||||
sudo('ufw allow 1936') # haproxy stats
|
||||
sudo('apt-get install -y haproxy')
|
||||
sudo('apt-get remove -y haproxy')
|
||||
with cd(env.VENDOR_PATH):
|
||||
|
|
@ -720,7 +727,7 @@ def setup_haproxy(debug=False):
|
|||
run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path))
|
||||
put('config/haproxy_rsyslog.conf', '/etc/rsyslog.d/49-haproxy.conf', use_sudo=True)
|
||||
sudo('restart rsyslog')
|
||||
|
||||
|
||||
sudo('/etc/init.d/haproxy stop')
|
||||
sudo('/etc/init.d/haproxy start')
|
||||
|
||||
|
|
@ -730,7 +737,7 @@ def config_haproxy(debug=False):
|
|||
else:
|
||||
put('../secrets-newsblur/configs/haproxy.conf', '/etc/haproxy/haproxy.cfg', use_sudo=True)
|
||||
sudo('/etc/init.d/haproxy reload')
|
||||
|
||||
|
||||
def upgrade_django():
|
||||
with cd(env.NEWSBLUR_PATH), settings(warn_only=True):
|
||||
sudo('supervisorctl stop gunicorn')
|
||||
|
|
@ -752,10 +759,10 @@ def downgrade_pil():
|
|||
sudo('rm -fr /usr/local/lib/python2.7/dist-packages/Pillow*')
|
||||
pull()
|
||||
kill()
|
||||
|
||||
|
||||
# ==============
|
||||
# = Setup - DB =
|
||||
# ==============
|
||||
# ==============
|
||||
|
||||
# @parallel
|
||||
def setup_db_firewall():
|
||||
|
|
@ -771,19 +778,19 @@ def setup_db_firewall():
|
|||
sudo('ufw default deny')
|
||||
sudo('ufw allow ssh')
|
||||
sudo('ufw allow 80')
|
||||
|
||||
|
||||
# DigitalOcean
|
||||
for ip in set(env.roledefs['app'] +
|
||||
env.roledefs['db'] +
|
||||
env.roledefs['dev'] +
|
||||
env.roledefs['debug'] +
|
||||
env.roledefs['task'] +
|
||||
for ip in set(env.roledefs['app'] +
|
||||
env.roledefs['db'] +
|
||||
env.roledefs['dev'] +
|
||||
env.roledefs['debug'] +
|
||||
env.roledefs['task'] +
|
||||
env.roledefs['node']):
|
||||
sudo('ufw allow proto tcp from %s to any port %s' % (
|
||||
ip,
|
||||
','.join(map(str, ports))
|
||||
))
|
||||
|
||||
|
||||
# EC2
|
||||
for host in set(env.roledefs['ec2task']):
|
||||
ip = re.search('ec2-(\d+-\d+-\d+-\d+)', host).group(1).replace('-', '.')
|
||||
|
|
@ -793,10 +800,10 @@ def setup_db_firewall():
|
|||
))
|
||||
|
||||
sudo('ufw --force enable')
|
||||
|
||||
|
||||
def setup_db_motd():
|
||||
put('config/motd_db.txt', '/etc/motd.tail', use_sudo=True)
|
||||
|
||||
|
||||
def setup_rabbitmq():
|
||||
sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list')
|
||||
run('wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc')
|
||||
|
|
@ -822,10 +829,10 @@ def setup_postgres(standby=False):
|
|||
# sudo('echo "%s" > /proc/sys/kernel/shmmax' % shmmax)
|
||||
# sudo('echo "\nkernel.shmmax = %s" > /etc/sysctl.conf' % shmmax)
|
||||
# sudo('sysctl -p')
|
||||
|
||||
|
||||
if standby:
|
||||
put('config/postgresql_recovery.conf', '/var/lib/postgresql/9.1/recovery.conf', use_sudo=True)
|
||||
|
||||
|
||||
sudo('/etc/init.d/postgresql stop')
|
||||
sudo('/etc/init.d/postgresql start')
|
||||
|
||||
|
|
@ -833,18 +840,18 @@ def copy_postgres_to_standby():
|
|||
slave = 'db12.newsblur.com'
|
||||
# Make sure you can ssh from master to slave and back.
|
||||
# Need to give postgres accounts keys in authroized_keys.
|
||||
|
||||
|
||||
# sudo('su postgres -c "psql -c \\"SELECT pg_start_backup(\'label\', true)\\""', pty=False)
|
||||
sudo('su postgres -c \"rsync -a --stats --progress /var/lib/postgresql/9.1/main postgres@%s:/var/lib/postgresql/9.1/ --exclude postmaster.pid\"' % slave, pty=False)
|
||||
# sudo('su postgres -c "psql -c \\"SELECT pg_stop_backup()\\""', pty=False)
|
||||
|
||||
|
||||
def setup_mongo():
|
||||
sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10')
|
||||
# sudo('echo "deb http://downloads.mongodb.org/distros/ubuntu 10.10 10gen" >> /etc/apt/sources.list.d/10gen.list')
|
||||
sudo('echo "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" >> /etc/apt/sources.list')
|
||||
sudo('apt-get update')
|
||||
sudo('apt-get -y install mongodb-10gen')
|
||||
put('config/mongodb.%s.conf' % ('prod' if env.user != 'ubuntu' else 'ec2'),
|
||||
put('config/mongodb.%s.conf' % ('prod' if env.user != 'ubuntu' else 'ec2'),
|
||||
'/etc/mongodb.conf', use_sudo=True)
|
||||
run('echo "ulimit -n 10000" > mongodb.defaults')
|
||||
sudo('mv mongodb.defaults /etc/default/mongodb')
|
||||
|
|
@ -907,8 +914,8 @@ def setup_munin():
|
|||
with settings(warn_only=True):
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_graph start')
|
||||
sudo('/etc/init.d/spawn_fcgi_munin_html start')
|
||||
|
||||
|
||||
|
||||
|
||||
def setup_db_munin():
|
||||
sudo('cp -frs %s/config/munin/mongo* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
|
||||
sudo('cp -frs %s/config/munin/pg_* /etc/munin/plugins/' % env.NEWSBLUR_PATH)
|
||||
|
|
@ -927,7 +934,7 @@ def enable_celerybeat():
|
|||
put('config/supervisor_celeryd_beat_feeds.conf', '/etc/supervisor/conf.d/celeryd_beat_feeds.conf', use_sudo=True)
|
||||
sudo('supervisorctl reread')
|
||||
sudo('supervisorctl update')
|
||||
|
||||
|
||||
def setup_db_mdadm():
|
||||
sudo('apt-get -y install xfsprogs mdadm')
|
||||
sudo('yes | mdadm --create /dev/md0 --level=0 -c256 --raid-devices=4 /dev/xvdf /dev/xvdg /dev/xvdh /dev/xvdi')
|
||||
|
|
@ -944,8 +951,8 @@ def setup_db_mdadm():
|
|||
def setup_original_page_server():
|
||||
setup_node()
|
||||
sudo('mkdir -p /srv/originals')
|
||||
sudo('chown sclay.sclay -R /srv/originals')
|
||||
put('config/supervisor_node_original.conf',
|
||||
sudo('chown %s.%s -R /srv/originals' % (env.user, env.user)) # We assume that the group is the same name as the user. It's common on linux
|
||||
put('config/supervisor_node_original.conf',
|
||||
'/etc/supervisor/conf.d/node_original.conf', use_sudo=True)
|
||||
sudo('supervisorctl reread')
|
||||
sudo('supervisorctl reload')
|
||||
|
|
@ -954,13 +961,13 @@ def setup_elasticsearch():
|
|||
ES_VERSION = "0.90.0"
|
||||
sudo('apt-get update')
|
||||
sudo('apt-get install openjdk-7-jre -y')
|
||||
|
||||
|
||||
with cd(env.VENDOR_PATH):
|
||||
run('mkdir elasticsearch-%s' % ES_VERSION)
|
||||
with cd(os.path.join(env.VENDOR_PATH, 'elasticsearch-%s' % ES_VERSION)):
|
||||
run('wget http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.deb' % ES_VERSION)
|
||||
sudo('dpkg -i elasticsearch-%s.deb' % ES_VERSION)
|
||||
|
||||
|
||||
# ================
|
||||
# = Setup - Task =
|
||||
# ================
|
||||
|
|
@ -973,13 +980,13 @@ def setup_task_firewall():
|
|||
|
||||
def setup_task_motd():
|
||||
put('config/motd_task.txt', '/etc/motd.tail', use_sudo=True)
|
||||
|
||||
|
||||
def enable_celery_supervisor(queue=None):
|
||||
if not queue:
|
||||
put('config/supervisor_celeryd.conf', '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True)
|
||||
else:
|
||||
put('config/supervisor_celeryd_%s.conf' % queue, '/etc/supervisor/conf.d/celeryd.conf', use_sudo=True)
|
||||
|
||||
|
||||
sudo('supervisorctl reread')
|
||||
sudo('supervisorctl update')
|
||||
|
||||
|
|
@ -992,7 +999,7 @@ def copy_task_settings():
|
|||
host = env.host.split('.', 2)[0]
|
||||
else:
|
||||
host = env.host_string.split('.', 2)[0]
|
||||
|
||||
|
||||
with settings(warn_only=True):
|
||||
put('../secrets-newsblur/settings/task_settings.py', '%s/local_settings.py' % env.NEWSBLUR_PATH)
|
||||
run('echo "\nSERVER_NAME = \\\\"%s\\\\"" >> %s/local_settings.py' % (host, env.NEWSBLUR_PATH))
|
||||
|
|
@ -1011,14 +1018,14 @@ def setup_do(name, size=2):
|
|||
region_id = doapi.regions()[0].id
|
||||
images = dict((s.name, s.id) for s in doapi.images())
|
||||
image_id = images[IMAGE_NAME]
|
||||
instance = doapi.create_droplet(name=name,
|
||||
size_id=size_id,
|
||||
image_id=image_id,
|
||||
region_id=region_id,
|
||||
instance = doapi.create_droplet(name=name,
|
||||
size_id=size_id,
|
||||
image_id=image_id,
|
||||
region_id=region_id,
|
||||
ssh_key_ids=[str(ssh_key_id)],
|
||||
virtio=True)
|
||||
print "Booting droplet: %s/%s (size: %s)" % (instance.id, IMAGE_NAME, INSTANCE_SIZE)
|
||||
|
||||
|
||||
instance = doapi.show_droplet(instance.id)
|
||||
i = 0
|
||||
while True:
|
||||
|
|
@ -1035,38 +1042,39 @@ def setup_do(name, size=2):
|
|||
else:
|
||||
print "!!! Error: %s" % instance.status
|
||||
return
|
||||
|
||||
|
||||
host = instance.ip_address
|
||||
env.host_string = host
|
||||
time.sleep(10)
|
||||
add_user_to_do()
|
||||
|
||||
|
||||
def add_user_to_do():
|
||||
env.user = "root"
|
||||
repo_user = "sclay"
|
||||
with settings(warn_only=True):
|
||||
run('useradd -m sclay')
|
||||
setup_sudoers("sclay")
|
||||
run('mkdir -p ~sclay/.ssh && chmod 700 ~sclay/.ssh')
|
||||
run('rm -fr ~sclay/.ssh/id_dsa*')
|
||||
run('ssh-keygen -t dsa -f ~sclay/.ssh/id_dsa -N ""')
|
||||
run('touch ~sclay/.ssh/authorized_keys')
|
||||
run('useradd -m %s' % (repo_user))
|
||||
setup_sudoers("%s" % (repo_user))
|
||||
run('mkdir -p ~%s/.ssh && chmod 700 ~%s/.ssh' % (repo_user, repo_user))
|
||||
run('rm -fr ~%s/.ssh/id_dsa*' % (repo_user))
|
||||
run('ssh-keygen -t dsa -f ~%s/.ssh/id_dsa -N ""' % (repo_user, repo_user))
|
||||
run('touch ~%s/.ssh/authorized_keys' % (repo_user, repo_user))
|
||||
put("~/.ssh/id_dsa.pub", "authorized_keys")
|
||||
run('echo `cat authorized_keys` >> ~sclay/.ssh/authorized_keys')
|
||||
run('echo `cat authorized_keys` >> ~%s/.ssh/authorized_keys' % (repo_user))
|
||||
run('rm authorized_keys')
|
||||
run('chown sclay.sclay -R ~sclay/.ssh')
|
||||
env.user = "sclay"
|
||||
run('chown %s.%s -R ~%s/.ssh' % (repo_user, repo_user, repo_user))
|
||||
env.user = repo_user
|
||||
|
||||
# ===============
|
||||
# = Setup - EC2 =
|
||||
# ===============
|
||||
|
||||
def setup_ec2():
|
||||
AMI_NAME = 'ami-834cf1ea' # Ubuntu 64-bit 12.04 LTS
|
||||
AMI_NAME = 'ami-834cf1ea' # Ubuntu 64-bit 12.04 LTS
|
||||
# INSTANCE_TYPE = 'c1.medium'
|
||||
INSTANCE_TYPE = 'c1.medium'
|
||||
conn = EC2Connection(django_settings.AWS_ACCESS_KEY_ID, django_settings.AWS_SECRET_ACCESS_KEY)
|
||||
reservation = conn.run_instances(AMI_NAME, instance_type=INSTANCE_TYPE,
|
||||
key_name='sclay',
|
||||
key_name=env.EC2_KEY_NAME,
|
||||
security_groups=['db-mongo'])
|
||||
instance = reservation.instances[0]
|
||||
print "Booting reservation: %s/%s (size: %s)" % (reservation, instance, INSTANCE_TYPE)
|
||||
|
|
@ -1085,12 +1093,12 @@ def setup_ec2():
|
|||
else:
|
||||
print "!!! Error: %s" % instance.state
|
||||
return
|
||||
|
||||
|
||||
host = instance.public_dns_name
|
||||
env.host_string = host
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ==============
|
||||
# = Tasks - DB =
|
||||
# ==============
|
||||
|
|
@ -1098,19 +1106,20 @@ def setup_ec2():
|
|||
def restore_postgres(port=5433):
|
||||
backup_date = '2013-01-29-09-00'
|
||||
yes = prompt("Dropping and creating NewsBlur PGSQL db. Sure?")
|
||||
if yes != 'y': return
|
||||
if yes != 'y':
|
||||
return
|
||||
# run('PYTHONPATH=%s python utils/backups/s3.py get backup_postgresql_%s.sql.gz' % (env.NEWSBLUR_PATH, backup_date))
|
||||
# sudo('su postgres -c "createuser -p %s -U newsblur"' % (port,))
|
||||
run('dropdb newsblur -p %s -U postgres' % (port,), pty=False)
|
||||
run('createdb newsblur -p %s -O newsblur' % (port,), pty=False)
|
||||
run('pg_restore -p %s --role=newsblur --dbname=newsblur /Users/sclay/Documents/backups/backup_postgresql_%s.sql.gz' % (port, backup_date), pty=False)
|
||||
|
||||
|
||||
def restore_mongo():
|
||||
backup_date = '2012-07-24-09-00'
|
||||
run('PYTHONPATH=/home/%s/newsblur python s3.py get backup_mongo_%s.tgz' % (env.user, backup_date))
|
||||
run('tar -xf backup_mongo_%s.tgz' % backup_date)
|
||||
run('mongorestore backup_mongo_%s' % backup_date)
|
||||
|
||||
|
||||
# ======
|
||||
# = S3 =
|
||||
# ======
|
||||
|
|
|
|||
20
settings.py
20
settings.py
|
|
@ -80,7 +80,7 @@ ALLOWED_HOSTS = ['*']
|
|||
|
||||
PRODUCTION = NEWSBLUR_DIR.find('/home/conesus/newsblur') == 0
|
||||
STAGING = NEWSBLUR_DIR.find('/home/conesus/staging') == 0
|
||||
DEVELOPMENT = NEWSBLUR_DIR.find('/Users/') == 0
|
||||
DEVELOPMENT = (NEWSBLUR_DIR.find('/Users/') == 0)
|
||||
|
||||
# ===========================
|
||||
# = Django-specific Modules =
|
||||
|
|
@ -118,7 +118,7 @@ LOGGING = {
|
|||
'disable_existing_loggers': False,
|
||||
'formatters': {
|
||||
'verbose': {
|
||||
'format': '[%(asctime)-12s] %(message)s',
|
||||
'format': '[%(asctime)-12s] %(message)s',
|
||||
'datefmt': '%b %d %H:%M:%S'
|
||||
},
|
||||
'simple': {
|
||||
|
|
@ -192,7 +192,7 @@ ROOT_URLCONF = 'urls'
|
|||
INTERNAL_IPS = ('127.0.0.1',)
|
||||
LOGGING_LOG_SQL = True
|
||||
APPEND_SLASH = False
|
||||
SOUTH_TESTS_MIGRATE = False
|
||||
SOUTH_TESTS_MIGRATE = False
|
||||
SESSION_ENGINE = "django.contrib.sessions.backends.db"
|
||||
TEST_RUNNER = "utils.testrunner.TestRunner"
|
||||
SESSION_COOKIE_NAME = 'newsblur_sessionid'
|
||||
|
|
@ -323,8 +323,8 @@ CELERY_QUEUES = {
|
|||
CELERY_DEFAULT_QUEUE = "work_queue"
|
||||
|
||||
CELERYD_PREFETCH_MULTIPLIER = 1
|
||||
CELERY_IMPORTS = ("apps.rss_feeds.tasks",
|
||||
"apps.social.tasks",
|
||||
CELERY_IMPORTS = ("apps.rss_feeds.tasks",
|
||||
"apps.social.tasks",
|
||||
"apps.reader.tasks",
|
||||
"apps.feed_import.tasks",
|
||||
"apps.statistics.tasks",)
|
||||
|
|
@ -398,22 +398,22 @@ MONGO_ANALYTICS_DB = {
|
|||
|
||||
class MasterSlaveRouter(object):
|
||||
"""A router that sets up a simple master/slave configuration"""
|
||||
|
||||
|
||||
def db_for_read(self, model, **hints):
|
||||
"Point all read operations to a random slave"
|
||||
return 'slave'
|
||||
|
||||
|
||||
def db_for_write(self, model, **hints):
|
||||
"Point all write operations to the master"
|
||||
return 'default'
|
||||
|
||||
|
||||
def allow_relation(self, obj1, obj2, **hints):
|
||||
"Allow any relation between two objects in the db pool"
|
||||
db_list = ('slave','default')
|
||||
if obj1._state.db in db_list and obj2._state.db in db_list:
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def allow_syncdb(self, db, model):
|
||||
"Explicitly put all models on all databases."
|
||||
return True
|
||||
|
|
@ -474,7 +474,7 @@ if not DEVELOPMENT:
|
|||
INSTALLED_APPS += (
|
||||
'gunicorn',
|
||||
'raven.contrib.django',
|
||||
'django_ses',
|
||||
'django_ses',
|
||||
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue