mirror of
https://github.com/viq/NewsBlur.git
synced 2025-04-13 09:38:09 +00:00
2413 lines
78 KiB
Python
2413 lines
78 KiB
Python
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from collections import defaultdict
|
|
from contextlib import contextmanager as _contextmanager
|
|
from pprint import pprint
|
|
|
|
import yaml
|
|
from boto.ec2.connection import EC2Connection
|
|
|
|
# from fabric.colors import red, green, blue, cyan, magenta, white, yellow
|
|
from boto.s3.connection import S3Connection
|
|
from boto.s3.key import Key
|
|
from fabric.api import (
|
|
cd,
|
|
env,
|
|
lcd,
|
|
local,
|
|
parallel,
|
|
prefix,
|
|
put,
|
|
run,
|
|
serial,
|
|
settings,
|
|
sudo,
|
|
)
|
|
from fabric.contrib import django, files
|
|
from fabric.operations import prompt
|
|
from fabric.state import connections
|
|
|
|
# django.setup()
|
|
|
|
try:
|
|
import digitalocean
|
|
except ImportError:
|
|
print("Digital Ocean's API not loaded. Install python-digitalocean.")
|
|
|
|
|
|
django.settings_module("newsblur_web.settings")
|
|
try:
|
|
from django.conf import settings as django_settings
|
|
except ImportError:
|
|
print(" ---> Django not installed yet.")
|
|
django_settings = None
|
|
|
|
# ============
|
|
# = DEFAULTS =
|
|
# ============
|
|
|
|
env.NEWSBLUR_PATH = "/srv/newsblur"
|
|
env.SECRETS_PATH = "/srv/secrets-newsblur"
|
|
env.VENDOR_PATH = "/srv/code"
|
|
env.user = "sclay"
|
|
env.key_filename = os.path.join(env.SECRETS_PATH, "keys/newsblur.key")
|
|
env.connection_attempts = 10
|
|
env.do_ip_to_hostname = {}
|
|
env.colorize_errors = True
|
|
|
|
# =========
|
|
# = Roles =
|
|
# =========
|
|
|
|
try:
|
|
hosts_path = os.path.expanduser(os.path.join(env.SECRETS_PATH, "configs/hosts.yml"))
|
|
roles = yaml.load(open(hosts_path))
|
|
for role_name, hosts in list(roles.items()):
|
|
if isinstance(hosts, dict):
|
|
roles[role_name] = [host for host in list(hosts.keys())]
|
|
env.roledefs = roles
|
|
except:
|
|
print(" ***> No role definitions found in %s. Using default roles." % hosts_path)
|
|
env.roledefs = {
|
|
"app": ["app01.newsblur.com"],
|
|
"db": ["db01.newsblur.com"],
|
|
"task": ["task01.newsblur.com"],
|
|
}
|
|
|
|
|
|
def do_roledefs(split=False, debug=False):
|
|
doapi = digitalocean.Manager(token=django_settings.DO_TOKEN_FABRIC)
|
|
droplets = doapi.get_all_droplets()
|
|
env.do_ip_to_hostname = {}
|
|
hostnames = {}
|
|
for droplet in droplets:
|
|
roledef = re.split(r"([0-9]+)", droplet.name)[0]
|
|
if roledef not in env.roledefs:
|
|
env.roledefs[roledef] = []
|
|
if roledef not in hostnames:
|
|
hostnames[roledef] = []
|
|
if droplet.ip_address not in hostnames[roledef]:
|
|
hostnames[roledef].append({"name": droplet.name, "address": droplet.ip_address})
|
|
env.do_ip_to_hostname[droplet.ip_address] = droplet.name
|
|
if droplet.ip_address not in env.roledefs[roledef]:
|
|
env.roledefs[roledef].append(droplet.ip_address)
|
|
|
|
if split:
|
|
return hostnames
|
|
return droplets
|
|
|
|
|
|
def list_do():
|
|
droplets = assign_digitalocean_roledefs(split=True)
|
|
pprint(droplets)
|
|
|
|
# Uncomment below to print all IP addresses
|
|
# for group in droplets.values():
|
|
# for server in group:
|
|
# if 'address' in server:
|
|
# print(server['address'])
|
|
|
|
doapi = digitalocean.Manager(token=django_settings.DO_TOKEN_FABRIC)
|
|
droplets = doapi.get_all_droplets()
|
|
sizes = doapi.get_all_sizes()
|
|
sizes = dict((size.slug, size.price_monthly) for size in sizes)
|
|
role_costs = defaultdict(int)
|
|
total_cost = 0
|
|
for droplet in droplets:
|
|
roledef = re.split(r"([0-9]+)", droplet.name)[0]
|
|
cost = droplet.size["price_monthly"]
|
|
role_costs[roledef] += cost
|
|
total_cost += cost
|
|
|
|
print("\n\n Costs:")
|
|
pprint(dict(role_costs))
|
|
print(" ---> Total cost: $%s/month" % total_cost)
|
|
|
|
|
|
def host(*names):
|
|
env.hosts = []
|
|
env.doname = ",".join(names)
|
|
hostnames = assign_digitalocean_roledefs(split=True)
|
|
for role, hosts in list(hostnames.items()):
|
|
for host in hosts:
|
|
if isinstance(host, dict) and host["name"] in names:
|
|
env.hosts.append(host["address"])
|
|
print(" ---> Using %s as hosts" % env.hosts)
|
|
|
|
|
|
# ================
|
|
# = Environments =
|
|
# ================
|
|
|
|
|
|
def server():
|
|
env.NEWSBLUR_PATH = "/srv/newsblur"
|
|
env.VENDOR_PATH = "/srv/code"
|
|
|
|
|
|
def assign_digitalocean_roledefs(split=False):
|
|
server()
|
|
droplets = do_roledefs(split=split)
|
|
if split:
|
|
for roledef, hosts in list(env.roledefs.items()):
|
|
if roledef not in droplets:
|
|
droplets[roledef] = hosts
|
|
|
|
return droplets
|
|
|
|
|
|
def app():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["app"]
|
|
|
|
|
|
def web():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["app", "push", "work", "search"]
|
|
|
|
|
|
def work():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["work"]
|
|
|
|
|
|
def www():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["www"]
|
|
|
|
|
|
def dev():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["dev"]
|
|
|
|
|
|
def debug():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["debug"]
|
|
|
|
|
|
def node():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["node"]
|
|
|
|
|
|
def push():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["push"]
|
|
|
|
|
|
def db():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["db", "search"]
|
|
|
|
|
|
def task():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["task"]
|
|
|
|
|
|
def ec2task():
|
|
ec2()
|
|
env.roles = ["ec2task"]
|
|
|
|
|
|
def ec2():
|
|
env.user = "ubuntu"
|
|
env.key_filename = ["/Users/sclay/.ec2/sclay.pem"]
|
|
assign_digitalocean_roledefs()
|
|
|
|
|
|
def all():
|
|
assign_digitalocean_roledefs()
|
|
env.roles = ["app", "db", "debug", "node", "push", "work", "www", "search"]
|
|
|
|
|
|
# =============
|
|
# = Bootstrap =
|
|
# =============
|
|
|
|
|
|
def setup_common():
|
|
setup_installs()
|
|
change_shell()
|
|
setup_user()
|
|
setup_sudoers()
|
|
setup_ulimit()
|
|
setup_do_monitoring()
|
|
setup_libxml()
|
|
setup_psql_client()
|
|
setup_repo()
|
|
setup_local_files()
|
|
setup_time_calibration()
|
|
setup_pip()
|
|
setup_virtualenv()
|
|
setup_repo_local_settings()
|
|
pip()
|
|
setup_supervisor()
|
|
setup_hosts()
|
|
setup_pgbouncer()
|
|
config_pgbouncer()
|
|
setup_mongoengine_repo()
|
|
# setup_forked_mongoengine()
|
|
# setup_pymongo_repo()
|
|
setup_logrotate()
|
|
copy_certificates()
|
|
setup_nginx()
|
|
setup_munin()
|
|
|
|
|
|
def setup_all():
|
|
setup_common()
|
|
setup_app(skip_common=True)
|
|
setup_db(skip_common=True)
|
|
setup_task(skip_common=True)
|
|
|
|
|
|
def setup_app_docker(skip_common=False):
|
|
if not skip_common:
|
|
setup_common()
|
|
setup_app_firewall()
|
|
setup_motd("app")
|
|
|
|
change_shell()
|
|
setup_user()
|
|
setup_sudoers()
|
|
setup_ulimit()
|
|
setup_do_monitoring()
|
|
setup_repo()
|
|
setup_local_files()
|
|
# setup_time_calibration()
|
|
|
|
setup_docker()
|
|
|
|
done()
|
|
sudo("reboot")
|
|
|
|
|
|
def setup_app(skip_common=False, node=False):
|
|
if not skip_common:
|
|
setup_common()
|
|
setup_app_firewall()
|
|
setup_motd("app")
|
|
copy_app_settings()
|
|
config_nginx()
|
|
setup_gunicorn(supervisor=True)
|
|
if node:
|
|
setup_node()
|
|
deploy_web()
|
|
config_monit_app()
|
|
setup_usage_monitor()
|
|
done()
|
|
sudo("reboot")
|
|
|
|
|
|
def setup_app_image():
|
|
copy_app_settings()
|
|
setup_hosts()
|
|
config_pgbouncer()
|
|
pull()
|
|
pip()
|
|
deploy_web()
|
|
done()
|
|
sudo("reboot")
|
|
|
|
|
|
def setup_node():
|
|
setup_node_app()
|
|
config_node(full=True)
|
|
|
|
|
|
def setup_db(engine=None, skip_common=False, skip_benchmark=False):
|
|
if not skip_common:
|
|
setup_common()
|
|
setup_db_firewall()
|
|
setup_motd("db")
|
|
copy_db_settings()
|
|
if engine == "postgres":
|
|
setup_postgres(standby=False)
|
|
setup_postgres_backups()
|
|
elif engine == "postgres_slave":
|
|
setup_postgres(standby=True)
|
|
elif engine and engine.startswith("mongo"):
|
|
setup_mongo()
|
|
# setup_mongo_mms()
|
|
setup_mongo_backups()
|
|
elif engine == "redis":
|
|
setup_redis()
|
|
setup_redis_backups()
|
|
setup_redis_monitor()
|
|
elif engine == "redis_slave":
|
|
setup_redis(slave=True)
|
|
setup_redis_monitor()
|
|
elif engine == "elasticsearch":
|
|
setup_elasticsearch()
|
|
setup_db_search()
|
|
setup_gunicorn(supervisor=False)
|
|
setup_db_munin()
|
|
setup_db_monitor()
|
|
setup_usage_monitor()
|
|
if not skip_benchmark:
|
|
benchmark()
|
|
done()
|
|
|
|
# if env.user == 'ubuntu':
|
|
# setup_db_mdadm()
|
|
|
|
|
|
def setup_task(queue=None, skip_common=False):
|
|
if not skip_common:
|
|
setup_common()
|
|
setup_task_firewall()
|
|
setup_motd("task")
|
|
copy_task_settings()
|
|
enable_celery_supervisor(queue)
|
|
setup_gunicorn(supervisor=False)
|
|
config_monit_task()
|
|
setup_usage_monitor()
|
|
done()
|
|
sudo("reboot")
|
|
|
|
|
|
def setup_task_image():
|
|
setup_installs()
|
|
copy_task_settings()
|
|
setup_hosts()
|
|
config_pgbouncer()
|
|
pull()
|
|
pip()
|
|
deploy(reload=True)
|
|
done()
|
|
sudo("reboot")
|
|
|
|
|
|
# ==================
|
|
# = Setup - Docker =
|
|
# ==================
|
|
|
|
|
|
def setup_docker():
|
|
packages = [
|
|
"build-essential",
|
|
]
|
|
sudo(
|
|
'DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install %s'
|
|
% " ".join(packages)
|
|
)
|
|
|
|
sudo("apt install -fy docker docker-compose")
|
|
sudo("usermod -aG docker ${USER}")
|
|
sudo("su - ${USER}")
|
|
|
|
copy_certificates()
|
|
|
|
|
|
# ==================
|
|
# = Setup - Common =
|
|
# ==================
|
|
|
|
|
|
def done():
|
|
print("\n\n\n\n-----------------------------------------------------")
|
|
print(
|
|
"\n\n %s / %s IS SUCCESSFULLY BOOTSTRAPPED"
|
|
% (env.get("doname") or env.host_string, env.host_string)
|
|
)
|
|
print("\n\n-----------------------------------------------------\n\n\n\n")
|
|
|
|
|
|
def setup_installs():
|
|
packages = [
|
|
"build-essential",
|
|
"gcc",
|
|
"scons",
|
|
"libreadline-dev",
|
|
"sysstat",
|
|
"iotop",
|
|
"git",
|
|
"python2",
|
|
"python2.7-dev",
|
|
"locate",
|
|
"software-properties-common",
|
|
"libpcre3-dev",
|
|
"libncurses5-dev",
|
|
"libdbd-pg-perl",
|
|
"libssl-dev",
|
|
"libffi-dev",
|
|
"libevent-dev",
|
|
"make",
|
|
"postgresql-common",
|
|
"ssl-cert",
|
|
"python-setuptools",
|
|
"libyaml-0-2",
|
|
"pgbouncer",
|
|
"python-yaml",
|
|
"python-numpy",
|
|
"curl",
|
|
"monit",
|
|
"ufw",
|
|
"libjpeg8",
|
|
"libjpeg62-dev",
|
|
"libfreetype6",
|
|
"libfreetype6-dev",
|
|
"libmysqlclient-dev",
|
|
"libblas-dev",
|
|
"liblapack-dev",
|
|
"libatlas-base-dev",
|
|
"gfortran",
|
|
"libpq-dev",
|
|
]
|
|
# sudo("sed -i -e 's/archive.ubuntu.com\|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list")
|
|
put("config/apt_sources.conf", "/etc/apt/sources.list", use_sudo=True)
|
|
run("sleep 10") # Dies on a lock, so just delay
|
|
sudo("apt-get -y update")
|
|
run("sleep 10") # Dies on a lock, so just delay
|
|
sudo(
|
|
'DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade'
|
|
)
|
|
run("sleep 10") # Dies on a lock, so just delay
|
|
sudo(
|
|
'DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install %s'
|
|
% " ".join(packages)
|
|
)
|
|
|
|
with settings(warn_only=True):
|
|
sudo("ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so /usr/lib")
|
|
sudo("ln -s /usr/lib/x86_64-linux-gnu/libfreetype.so /usr/lib")
|
|
sudo("ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib")
|
|
|
|
with settings(warn_only=True):
|
|
sudo("mkdir -p %s" % env.VENDOR_PATH)
|
|
sudo("chown %s.%s %s" % (env.user, env.user, env.VENDOR_PATH))
|
|
|
|
|
|
def change_shell():
|
|
sudo("apt-get -fy install zsh")
|
|
with settings(warn_only=True):
|
|
run("git clone git://github.com/robbyrussell/oh-my-zsh.git ~/.oh-my-zsh")
|
|
run(
|
|
"git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting"
|
|
)
|
|
sudo("chsh %s -s /bin/zsh" % env.user)
|
|
|
|
|
|
def setup_user():
|
|
# run('useradd -c "NewsBlur" -m newsblur -s /bin/zsh')
|
|
# run('openssl rand -base64 8 | tee -a ~conesus/.password | passwd -stdin conesus')
|
|
run("mkdir -p ~/.ssh && chmod 700 ~/.ssh")
|
|
run("rm -fr ~/.ssh/id_dsa*")
|
|
run('ssh-keygen -t dsa -f ~/.ssh/id_dsa -N ""')
|
|
run("touch ~/.ssh/authorized_keys")
|
|
put("~/.ssh/id_dsa.pub", "authorized_keys")
|
|
run('echo "\n" >> ~sclay/.ssh/authorized_keys')
|
|
run("echo `cat authorized_keys` >> ~sclay/.ssh/authorized_keys")
|
|
run("rm authorized_keys")
|
|
|
|
|
|
def copy_ssh_keys(username="sclay", private=False):
|
|
sudo("mkdir -p ~%s/.ssh" % username)
|
|
|
|
put(os.path.join(env.SECRETS_PATH, "keys/newsblur.key.pub"), "local.key.pub")
|
|
sudo("mv local.key.pub ~%s/.ssh/id_rsa.pub" % username)
|
|
if private:
|
|
put(os.path.join(env.SECRETS_PATH, "keys/newsblur.key"), "local.key")
|
|
sudo("mv local.key ~%s/.ssh/id_rsa" % username)
|
|
|
|
sudo('echo "\n" >> ~%s/.ssh/authorized_keys' % username)
|
|
sudo("echo `cat ~%s/.ssh/id_rsa.pub` >> ~%s/.ssh/authorized_keys" % (username, username))
|
|
sudo("chown -R %s.%s ~%s/.ssh" % (username, username, username))
|
|
sudo("chmod 700 ~%s/.ssh" % username)
|
|
sudo("chmod 600 ~%s/.ssh/id_rsa*" % username)
|
|
|
|
|
|
def setup_repo():
|
|
sudo("mkdir -p /srv")
|
|
sudo("chown -R %s.%s /srv" % (env.user, env.user))
|
|
with settings(warn_only=True):
|
|
run("git clone https://github.com/samuelclay/NewsBlur.git %s" % env.NEWSBLUR_PATH)
|
|
with settings(warn_only=True):
|
|
sudo("ln -sfn /srv/code /home/%s/code" % env.user)
|
|
sudo("ln -sfn /srv/newsblur /home/%s/newsblur" % env.user)
|
|
|
|
|
|
def setup_repo_local_settings():
|
|
with virtualenv():
|
|
run("cp newsblur/local_settings.py.template newsblur/local_settings.py")
|
|
run("mkdir -p logs")
|
|
run("touch logs/newsblur.log")
|
|
|
|
|
|
def setup_local_files():
|
|
run("mkdir -p ~/.config/procps")
|
|
put("config/toprc", "~/.config/procps/toprc")
|
|
run("rm -f ~/.toprc")
|
|
put("config/zshrc", "~/.zshrc")
|
|
put("config/gitconfig.txt", "~/.gitconfig")
|
|
put("config/ssh.conf", "~/.ssh/config")
|
|
|
|
|
|
def setup_psql_client():
|
|
sudo("apt-get -y install postgresql-client")
|
|
sudo("mkdir -p /var/run/postgresql")
|
|
with settings(warn_only=True):
|
|
sudo("chown postgres.postgres /var/run/postgresql")
|
|
|
|
|
|
def setup_libxml():
|
|
sudo("apt-get -y install libxml2-dev libxslt1-dev python-lxml")
|
|
|
|
|
|
def setup_libxml_code():
|
|
with cd(env.VENDOR_PATH):
|
|
run("git clone git://git.gnome.org/libxml2")
|
|
run("git clone git://git.gnome.org/libxslt")
|
|
|
|
with cd(os.path.join(env.VENDOR_PATH, "libxml2")):
|
|
run("./configure && make && sudo make install")
|
|
|
|
with cd(os.path.join(env.VENDOR_PATH, "libxslt")):
|
|
run("./configure && make && sudo make install")
|
|
|
|
|
|
def setup_psycopg():
|
|
sudo("easy_install -U psycopg2")
|
|
|
|
|
|
def setup_virtualenv():
|
|
sudo("rm -fr ~/.cache") # Clean `sudo pip`
|
|
sudo("pip install --upgrade virtualenv")
|
|
sudo("pip install --upgrade virtualenvwrapper")
|
|
setup_local_files()
|
|
with prefix("WORKON_HOME=%s" % os.path.join(env.NEWSBLUR_PATH, "venv")):
|
|
with prefix("source /usr/local/bin/virtualenvwrapper.sh"):
|
|
with cd(env.NEWSBLUR_PATH):
|
|
# sudo('rmvirtualenv newsblur')
|
|
# sudo('rm -fr venv')
|
|
with settings(warn_only=True):
|
|
run("mkvirtualenv newsblur")
|
|
# run('echo "import sys; sys.setdefaultencoding(\'utf-8\')" | sudo tee venv/newsblur/lib/python2.7/sitecustomize.py')
|
|
# run('echo "/srv/newsblur" | sudo tee venv/newsblur/lib/python2.7/site-packages/newsblur.pth')
|
|
|
|
|
|
@_contextmanager
|
|
def virtualenv():
|
|
with prefix("WORKON_HOME=%s" % os.path.join(env.NEWSBLUR_PATH, "venv")):
|
|
with prefix("source /usr/local/bin/virtualenvwrapper.sh"):
|
|
with cd(env.NEWSBLUR_PATH):
|
|
with prefix("workon newsblur"):
|
|
yield
|
|
|
|
|
|
def setup_pip():
|
|
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
|
run("curl https://bootstrap.pypa.io/2.6/get-pip.py | sudo python2")
|
|
# sudo('python2 get-pip.py')
|
|
|
|
|
|
@parallel
|
|
def pip():
|
|
role = role_for_host()
|
|
|
|
pull()
|
|
with virtualenv():
|
|
if role == "task":
|
|
with settings(warn_only=True):
|
|
sudo("fallocate -l 4G /swapfile")
|
|
sudo("chmod 600 /swapfile")
|
|
sudo("mkswap /swapfile")
|
|
sudo("swapon /swapfile")
|
|
sudo("chown %s.%s -R %s" % (env.user, env.user, os.path.join(env.NEWSBLUR_PATH, "venv")))
|
|
# run('easy_install -U pip')
|
|
# run('pip install --upgrade pip')
|
|
# run('pip install --upgrade setuptools')
|
|
run("pip install -r requirements.txt")
|
|
if role == "task":
|
|
with settings(warn_only=True):
|
|
sudo("swapoff /swapfile")
|
|
|
|
|
|
def solo_pip(role):
|
|
if role == "app":
|
|
gunicorn_stop()
|
|
pip()
|
|
deploy_code(reload=True)
|
|
elif role == "task":
|
|
celery_stop()
|
|
copy_task_settings()
|
|
pip()
|
|
celery()
|
|
|
|
|
|
def setup_supervisor():
|
|
sudo("apt-get update")
|
|
sudo("apt-get -y install supervisor")
|
|
put("config/supervisord.conf", "/etc/supervisor/supervisord.conf", use_sudo=True)
|
|
sudo("/etc/init.d/supervisor stop")
|
|
sudo("sleep 2")
|
|
sudo("ulimit -n 100000 && /etc/init.d/supervisor start")
|
|
sudo("/usr/sbin/update-rc.d -f supervisor defaults")
|
|
sudo("systemctl enable supervisor")
|
|
sudo("systemctl start supervisor")
|
|
|
|
|
|
@parallel
|
|
def setup_hosts():
|
|
put(os.path.join(env.SECRETS_PATH, "configs/hosts"), "/etc/hosts", use_sudo=True)
|
|
sudo('echo "\n\n127.0.0.1 `hostname`" | sudo tee -a /etc/hosts')
|
|
|
|
|
|
def setup_pgbouncer():
|
|
sudo("apt-get remove -y pgbouncer")
|
|
sudo("apt-get install -y libevent-dev pkg-config libc-ares2 libc-ares-dev")
|
|
PGBOUNCER_VERSION = "1.15.0"
|
|
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
|
run(
|
|
"wget https://pgbouncer.github.io/downloads/files/%s/pgbouncer-%s.tar.gz"
|
|
% (PGBOUNCER_VERSION, PGBOUNCER_VERSION)
|
|
)
|
|
run("tar -xzf pgbouncer-%s.tar.gz" % PGBOUNCER_VERSION)
|
|
run("rm pgbouncer-%s.tar.gz" % PGBOUNCER_VERSION)
|
|
with cd("pgbouncer-%s" % PGBOUNCER_VERSION):
|
|
run("./configure --prefix=/usr/local")
|
|
run("make")
|
|
sudo("make install")
|
|
sudo("ln -s /usr/local/bin/pgbouncer /usr/sbin/pgbouncer")
|
|
config_pgbouncer()
|
|
|
|
|
|
def config_pgbouncer():
|
|
sudo("mkdir -p /etc/pgbouncer")
|
|
put("config/pgbouncer.conf", "pgbouncer.conf")
|
|
sudo("mv pgbouncer.conf /etc/pgbouncer/pgbouncer.ini")
|
|
put(os.path.join(env.SECRETS_PATH, "configs/pgbouncer_auth.conf"), "userlist.txt")
|
|
sudo("mv userlist.txt /etc/pgbouncer/userlist.txt")
|
|
sudo('echo "START=1" | sudo tee /etc/default/pgbouncer')
|
|
# sudo('su postgres -c "/etc/init.d/pgbouncer stop"', pty=False)
|
|
with settings(warn_only=True):
|
|
sudo("/etc/init.d/pgbouncer stop")
|
|
sudo("pkill -9 pgbouncer -e")
|
|
run("sleep 2")
|
|
sudo("/etc/init.d/pgbouncer start", pty=False)
|
|
|
|
|
|
@parallel
|
|
def kill_pgbouncer(stop=False):
|
|
# sudo('su postgres -c "/etc/init.d/pgbouncer stop"', pty=False)
|
|
with settings(warn_only=True):
|
|
sudo("/etc/init.d/pgbouncer stop")
|
|
run("sleep 2")
|
|
sudo("rm /var/log/postgresql/pgbouncer.pid")
|
|
with settings(warn_only=True):
|
|
sudo("pkill -9 pgbouncer")
|
|
run("sleep 2")
|
|
if not stop:
|
|
run("sudo /etc/init.d/pgbouncer start", pty=False)
|
|
|
|
|
|
def config_monit_task():
|
|
put("config/monit_task.conf", "/etc/monit/conf.d/celery.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def config_monit_node():
|
|
put("config/monit_node.conf", "/etc/monit/conf.d/node.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def config_monit_original():
|
|
put("config/monit_original.conf", "/etc/monit/conf.d/node_original.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def config_monit_app():
|
|
put("config/monit_app.conf", "/etc/monit/conf.d/gunicorn.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def config_monit_work():
|
|
put("config/monit_work.conf", "/etc/monit/conf.d/work.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def config_monit_redis():
|
|
sudo("chown root.root /etc/init.d/redis")
|
|
sudo("chmod a+x /etc/init.d/redis")
|
|
put("config/monit_debug.sh", "/etc/monit/monit_debug.sh", use_sudo=True)
|
|
sudo("chmod a+x /etc/monit/monit_debug.sh")
|
|
put("config/monit_redis.conf", "/etc/monit/conf.d/redis.conf", use_sudo=True)
|
|
sudo('echo "START=yes" | sudo tee /etc/default/monit')
|
|
sudo("/etc/init.d/monit restart")
|
|
|
|
|
|
def setup_mongoengine_repo():
|
|
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
|
run("rm -fr mongoengine")
|
|
run("git clone https://github.com/MongoEngine/mongoengine.git")
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine")
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/mongoengine-*")
|
|
sudo(
|
|
"ln -sfn %s /usr/local/lib/python2.7/dist-packages/mongoengine"
|
|
% os.path.join(env.VENDOR_PATH, "mongoengine/mongoengine")
|
|
)
|
|
with cd(os.path.join(env.VENDOR_PATH, "mongoengine")), settings(warn_only=True):
|
|
run("git co v0.8.2")
|
|
|
|
|
|
def clear_pymongo_repo():
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/pymongo*")
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/bson*")
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/gridfs*")
|
|
|
|
|
|
def setup_pymongo_repo():
|
|
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
|
run("git clone git://github.com/mongodb/mongo-python-driver.git pymongo")
|
|
# with cd(os.path.join(env.VENDOR_PATH, 'pymongo')):
|
|
# sudo('python setup.py install')
|
|
clear_pymongo_repo()
|
|
sudo(
|
|
"ln -sfn %s /usr/local/lib/python2.7/dist-packages/"
|
|
% os.path.join(env.VENDOR_PATH, "pymongo/{pymongo,bson,gridfs}")
|
|
)
|
|
|
|
|
|
def setup_forked_mongoengine():
|
|
with cd(os.path.join(env.VENDOR_PATH, "mongoengine")), settings(warn_only=True):
|
|
run("git remote add clay https://github.com/samuelclay/mongoengine.git")
|
|
run("git pull")
|
|
run("git fetch clay")
|
|
run("git checkout -b clay_master clay/master")
|
|
|
|
|
|
def switch_forked_mongoengine():
|
|
with cd(os.path.join(env.VENDOR_PATH, "mongoengine")):
|
|
run("git co dev")
|
|
run("git pull %s dev --force" % env.user)
|
|
# run('git checkout .')
|
|
# run('git checkout master')
|
|
# run('get branch -D dev')
|
|
# run('git checkout -b dev origin/dev')
|
|
|
|
|
|
def setup_logrotate(clear=True):
|
|
if clear:
|
|
run("find /srv/newsblur/logs/*.log | xargs tee")
|
|
with settings(warn_only=True):
|
|
sudo("find /var/log/mongodb/*.log | xargs tee")
|
|
put("config/logrotate.conf", "/etc/logrotate.d/newsblur", use_sudo=True)
|
|
put("config/logrotate.mongo.conf", "/etc/logrotate.d/mongodb", use_sudo=True)
|
|
put("config/logrotate.nginx.conf", "/etc/logrotate.d/nginx", use_sudo=True)
|
|
sudo("chown root.root /etc/logrotate.d/{newsblur,mongodb,nginx}")
|
|
sudo("chmod 644 /etc/logrotate.d/{newsblur,mongodb,nginx}")
|
|
with settings(warn_only=True):
|
|
sudo("chown sclay.sclay /srv/newsblur/logs/*.log")
|
|
sudo("logrotate -f /etc/logrotate.d/newsblur")
|
|
sudo("logrotate -f /etc/logrotate.d/nginx")
|
|
sudo("logrotate -f /etc/logrotate.d/mongodb")
|
|
|
|
|
|
def setup_ulimit():
|
|
# Increase File Descriptor limits.
|
|
run("export FILEMAX=`sysctl -n fs.file-max`", pty=False)
|
|
sudo("mv /etc/security/limits.conf /etc/security/limits.conf.bak", pty=False)
|
|
sudo("touch /etc/security/limits.conf", pty=False)
|
|
run('echo "root soft nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False)
|
|
run('echo "root hard nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False)
|
|
run('echo "* soft nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False)
|
|
run('echo "* hard nofile 100090\n" | sudo tee -a /etc/security/limits.conf', pty=False)
|
|
run('echo "fs.file-max = 100000\n" | sudo tee -a /etc/sysctl.conf', pty=False)
|
|
sudo("sysctl -p")
|
|
sudo("ulimit -n 100000")
|
|
connections.connect(env.host_string)
|
|
|
|
# run('touch /home/ubuntu/.bash_profile')
|
|
# run('echo "ulimit -n $FILEMAX" >> /home/ubuntu/.bash_profile')
|
|
|
|
# Increase Ephemeral Ports.
|
|
# sudo chmod 666 /etc/sysctl.conf
|
|
# echo "net.ipv4.ip_local_port_range = 1024 65535" >> /etc/sysctl.conf
|
|
# sudo chmod 644 /etc/sysctl.conf
|
|
|
|
|
|
def setup_do_monitoring():
|
|
run("curl -sSL https://agent.digitalocean.com/install.sh | sh")
|
|
|
|
|
|
def setup_syncookies():
|
|
sudo("echo 1 | sudo tee /proc/sys/net/ipv4/tcp_syncookies")
|
|
sudo("sudo /sbin/sysctl -w net.ipv4.tcp_syncookies=1")
|
|
|
|
|
|
def setup_sudoers(user=None):
|
|
sudo('echo "%s ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/sclay' % (user or env.user))
|
|
sudo("chmod 0440 /etc/sudoers.d/sclay")
|
|
|
|
|
|
def setup_nginx():
|
|
NGINX_VERSION = "1.19.5"
|
|
with cd(env.VENDOR_PATH), settings(warn_only=True):
|
|
sudo("groupadd nginx")
|
|
sudo("useradd -g nginx -d /var/www/htdocs -s /bin/false nginx")
|
|
run("wget http://nginx.org/download/nginx-%s.tar.gz" % NGINX_VERSION)
|
|
run("tar -xzf nginx-%s.tar.gz" % NGINX_VERSION)
|
|
run("rm nginx-%s.tar.gz" % NGINX_VERSION)
|
|
with cd("nginx-%s" % NGINX_VERSION):
|
|
run(
|
|
"./configure --with-http_ssl_module --with-http_stub_status_module --with-http_gzip_static_module --with-http_realip_module "
|
|
)
|
|
run("make")
|
|
sudo("make install")
|
|
config_nginx()
|
|
|
|
|
|
def config_nginx():
|
|
put("config/nginx.conf", "/usr/local/nginx/conf/nginx.conf", use_sudo=True)
|
|
sudo("mkdir -p /usr/local/nginx/conf/sites-enabled")
|
|
sudo("mkdir -p /var/log/nginx")
|
|
put("config/nginx.newsblur.conf", "/usr/local/nginx/conf/sites-enabled/newsblur.conf", use_sudo=True)
|
|
put("config/nginx-init", "/etc/init.d/nginx", use_sudo=True)
|
|
sudo("sed -i -e s/nginx_none/`cat /etc/hostname`/g /usr/local/nginx/conf/sites-enabled/newsblur.conf")
|
|
sudo("chmod 0755 /etc/init.d/nginx")
|
|
sudo("/usr/sbin/update-rc.d -f nginx defaults")
|
|
sudo("/etc/init.d/nginx restart")
|
|
copy_certificates()
|
|
|
|
|
|
# ===============
|
|
# = Setup - App =
|
|
# ===============
|
|
|
|
|
|
def setup_app_firewall():
|
|
sudo("ufw default deny")
|
|
sudo("ufw allow ssh") # ssh
|
|
sudo("ufw allow 80") # http
|
|
sudo("ufw allow 8000") # gunicorn
|
|
sudo("ufw allow 8888") # socket.io
|
|
sudo("ufw allow 8889") # socket.io ssl
|
|
sudo("ufw allow 443") # https
|
|
sudo("ufw --force enable")
|
|
|
|
|
|
def remove_gunicorn():
|
|
with cd(env.VENDOR_PATH):
|
|
sudo("rm -fr gunicorn")
|
|
|
|
|
|
def setup_gunicorn(supervisor=True, restart=True):
|
|
if supervisor:
|
|
put("config/supervisor_gunicorn.conf", "/etc/supervisor/conf.d/gunicorn.conf", use_sudo=True)
|
|
sudo("supervisorctl reread")
|
|
if restart:
|
|
sudo("supervisorctl update")
|
|
# with cd(env.VENDOR_PATH):
|
|
# sudo('rm -fr gunicorn')
|
|
# run('git clone git://github.com/benoitc/gunicorn.git')
|
|
# with cd(os.path.join(env.VENDOR_PATH, 'gunicorn')):
|
|
# run('git pull')
|
|
# sudo('python setup.py develop')
|
|
|
|
|
|
def update_gunicorn():
|
|
with cd(os.path.join(env.VENDOR_PATH, "gunicorn")):
|
|
run("git pull")
|
|
sudo("python setup.py develop")
|
|
|
|
|
|
def setup_staging():
|
|
run("git clone https://github.com/samuelclay/NewsBlur.git staging")
|
|
with cd("~/staging"):
|
|
run("cp ../newsblur/local_settings.py local_settings.py")
|
|
run("mkdir -p logs")
|
|
run("touch logs/newsblur.log")
|
|
|
|
|
|
def setup_node_app():
|
|
sudo("curl -sL https://deb.nodesource.com/setup_14.x | sudo bash -")
|
|
sudo("apt-get install -y nodejs")
|
|
# run('curl -L https://npmjs.org/install.sh | sudo sh')
|
|
# sudo('apt-get install npm')
|
|
sudo("sudo npm install -g npm")
|
|
sudo("npm install -g supervisor")
|
|
sudo("ufw allow 8888")
|
|
sudo("ufw allow 4040")
|
|
|
|
|
|
def config_node(full=False):
|
|
sudo("rm -f /etc/supervisor/conf.d/gunicorn.conf")
|
|
sudo("rm -f /etc/supervisor/conf.d/node.conf")
|
|
put("config/supervisor_node_unread.conf", "/etc/supervisor/conf.d/node_unread.conf", use_sudo=True)
|
|
put(
|
|
"config/supervisor_node_unread_ssl.conf", "/etc/supervisor/conf.d/node_unread_ssl.conf", use_sudo=True
|
|
)
|
|
put("config/supervisor_node_favicons.conf", "/etc/supervisor/conf.d/node_favicons.conf", use_sudo=True)
|
|
put("config/supervisor_node_text.conf", "/etc/supervisor/conf.d/node_text.conf", use_sudo=True)
|
|
|
|
if full:
|
|
run("rm -fr /srv/newsblur/node/node_modules")
|
|
with cd(os.path.join(env.NEWSBLUR_PATH, "node")):
|
|
run("npm install")
|
|
|
|
sudo("supervisorctl reload")
|
|
|
|
|
|
@parallel
|
|
def copy_app_settings():
|
|
run("rm -f %s/local_settings.py" % env.NEWSBLUR_PATH)
|
|
put(
|
|
os.path.join(env.SECRETS_PATH, "settings/app_settings.py"),
|
|
"%s/newsblur/local_settings.py" % env.NEWSBLUR_PATH,
|
|
)
|
|
run('echo "\nSERVER_NAME = \\\\"`hostname`\\\\"" >> %s/newsblur/local_settings.py' % env.NEWSBLUR_PATH)
|
|
|
|
|
|
def assemble_certificates():
|
|
with lcd(os.path.join(env.SECRETS_PATH, "certificates/comodo")):
|
|
local("pwd")
|
|
local(
|
|
"cat STAR_newsblur_com.crt EssentialSSLCA_2.crt ComodoUTNSGCCA.crt UTNAddTrustSGCCA.crt AddTrustExternalCARoot.crt > newsblur.com.crt"
|
|
)
|
|
|
|
|
|
def copy_certificates(copy=False):
|
|
cert_path = os.path.join(env.NEWSBLUR_PATH, "config/certificates")
|
|
run("mkdir -p %s" % cert_path)
|
|
fullchain_path = "/etc/letsencrypt/live/newsblur.com/fullchain.pem"
|
|
privkey_path = "/etc/letsencrypt/live/newsblur.com/privkey.pem"
|
|
|
|
if copy:
|
|
sudo("mkdir -p %s" % os.path.dirname(fullchain_path))
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/newsblur.com.pem"), fullchain_path, use_sudo=True)
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/newsblur.com.key"), privkey_path, use_sudo=True)
|
|
|
|
run("ln -fs %s %s" % (fullchain_path, os.path.join(cert_path, "newsblur.com.crt")))
|
|
run(
|
|
"ln -fs %s %s" % (fullchain_path, os.path.join(cert_path, "newsblur.com.pem"))
|
|
) # For backwards compatibility with hard-coded nginx configs
|
|
run("ln -fs %s %s" % (privkey_path, os.path.join(cert_path, "newsblur.com.key")))
|
|
run("ln -fs %s %s" % (privkey_path, os.path.join(cert_path, "newsblur.com.crt.key"))) # HAProxy
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/comodo/dhparams.pem"), cert_path)
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/ios/aps_development.pem"), cert_path)
|
|
|
|
# Export aps.cer from Apple issued certificate using Keychain Assistant
|
|
# openssl x509 -in aps.cer -inform DER -outform PEM -out aps.pem
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/ios/aps.pem"), cert_path)
|
|
# Export aps.p12 from aps.cer using Keychain Assistant
|
|
# openssl pkcs12 -in aps.p12 -out aps.p12.pem -nodes
|
|
put(os.path.join(env.SECRETS_PATH, "certificates/ios/aps.p12.pem"), cert_path)
|
|
|
|
|
|
def setup_certbot():
|
|
sudo("snap install --classic certbot")
|
|
sudo("snap set certbot trust-plugin-with-root=ok")
|
|
sudo("snap install certbot-dns-dnsimple")
|
|
sudo("ln -fs /snap/bin/certbot /usr/bin/certbot")
|
|
put(
|
|
os.path.join(env.SECRETS_PATH, "configs/certbot.conf"),
|
|
os.path.join(env.NEWSBLUR_PATH, "certbot.conf"),
|
|
)
|
|
sudo("chmod 0600 %s" % os.path.join(env.NEWSBLUR_PATH, "certbot.conf"))
|
|
sudo(
|
|
"certbot certonly -n --agree-tos "
|
|
" --dns-dnsimple --dns-dnsimple-credentials %s"
|
|
" --email samuel@newsblur.com --domains newsblur.com "
|
|
' -d "*.newsblur.com" -d "popular.global.newsblur.com"'
|
|
% (os.path.join(env.NEWSBLUR_PATH, "certbot.conf"))
|
|
)
|
|
sudo("chmod 0755 /etc/letsencrypt/{live,archive}")
|
|
sudo("chmod 0755 /etc/letsencrypt/archive/newsblur.com/privkey1.pem")
|
|
|
|
|
|
# def setup_certbot_old():
|
|
# sudo('add-apt-repository -y universe')
|
|
# sudo('add-apt-repository -y ppa:certbot/certbot')
|
|
# sudo('apt-get update')
|
|
# sudo('apt-get install -y certbot')
|
|
# sudo('apt-get install -y python3-certbot-dns-dnsimple')
|
|
# put(os.path.join(env.SECRETS_PATH, 'configs/certbot.conf'),
|
|
# os.path.join(env.NEWSBLUR_PATH, 'certbot.conf'))
|
|
# sudo('chmod 0600 %s' % os.path.join(env.NEWSBLUR_PATH, 'certbot.conf'))
|
|
# sudo('certbot certonly -n --agree-tos '
|
|
# ' --dns-dnsimple --dns-dnsimple-credentials %s'
|
|
# ' --email samuel@newsblur.com --domains newsblur.com '
|
|
# ' -d "*.newsblur.com" -d "global.popular.newsblur.com"' %
|
|
# (os.path.join(env.NEWSBLUR_PATH, 'certbot.conf')))
|
|
# sudo('chmod 0755 /etc/letsencrypt/{live,archive}')
|
|
# sudo('chmod 0755 /etc/letsencrypt/archive/newsblur.com/privkey1.pem')
|
|
|
|
|
|
@parallel
|
|
def maintenance_on():
|
|
role = role_for_host()
|
|
if role in ["work", "search"]:
|
|
sudo("supervisorctl stop all")
|
|
else:
|
|
put("templates/maintenance_off.html", "%s/templates/maintenance_off.html" % env.NEWSBLUR_PATH)
|
|
with virtualenv():
|
|
run("mv templates/maintenance_off.html templates/maintenance_on.html")
|
|
|
|
|
|
@parallel
|
|
def maintenance_off():
|
|
role = role_for_host()
|
|
if role in ["work", "search"]:
|
|
sudo("supervisorctl start all")
|
|
else:
|
|
with virtualenv():
|
|
run("mv templates/maintenance_on.html templates/maintenance_off.html")
|
|
run("git checkout templates/maintenance_off.html")
|
|
|
|
|
|
def setup_haproxy(debug=False):
|
|
version = "2.3.3"
|
|
sudo("ufw allow 81") # nginx moved
|
|
sudo("ufw allow 1936") # haproxy stats
|
|
# sudo('apt-get install -y haproxy')
|
|
# sudo('apt-get remove -y haproxy')
|
|
with cd(env.VENDOR_PATH):
|
|
run("wget http://www.haproxy.org/download/2.3/src/haproxy-%s.tar.gz" % version)
|
|
run("tar -xf haproxy-%s.tar.gz" % version)
|
|
with cd("haproxy-%s" % version):
|
|
run("make TARGET=linux-glibc USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1")
|
|
sudo("make install")
|
|
put("config/haproxy-init", "/etc/init.d/haproxy", use_sudo=True)
|
|
sudo("chmod u+x /etc/init.d/haproxy")
|
|
sudo("mkdir -p /etc/haproxy")
|
|
if debug:
|
|
put("config/debug_haproxy.conf", "/etc/haproxy/haproxy.cfg", use_sudo=True)
|
|
else:
|
|
build_haproxy()
|
|
put(os.path.join(env.SECRETS_PATH, "configs/haproxy.conf"), "/etc/haproxy/haproxy.cfg", use_sudo=True)
|
|
sudo('echo "ENABLED=1" | sudo tee /etc/default/haproxy')
|
|
cert_path = "%s/config/certificates" % env.NEWSBLUR_PATH
|
|
run("cat %s/newsblur.com.crt > %s/newsblur.pem" % (cert_path, cert_path))
|
|
run("cat %s/newsblur.com.key >> %s/newsblur.pem" % (cert_path, cert_path))
|
|
run("ln -s %s/newsblur.com.key %s/newsblur.pem.key" % (cert_path, cert_path))
|
|
put("config/haproxy_rsyslog.conf", "/etc/rsyslog.d/49-haproxy.conf", use_sudo=True)
|
|
# sudo('restart rsyslog')
|
|
sudo("update-rc.d -f haproxy defaults")
|
|
|
|
sudo("/etc/init.d/haproxy stop")
|
|
run("sleep 5")
|
|
sudo("/etc/init.d/haproxy start")
|
|
|
|
|
|
def config_haproxy(debug=False):
|
|
if debug:
|
|
put("config/debug_haproxy.conf", "/etc/haproxy/haproxy.cfg", use_sudo=True)
|
|
else:
|
|
build_haproxy()
|
|
put(os.path.join(env.SECRETS_PATH, "configs/haproxy.conf"), "/etc/haproxy/haproxy.cfg", use_sudo=True)
|
|
|
|
haproxy_check = run("haproxy -c -f /etc/haproxy/haproxy.cfg")
|
|
if haproxy_check.return_code == 0:
|
|
sudo("/etc/init.d/haproxy reload")
|
|
else:
|
|
print(" !!!> Uh-oh, HAProxy config doesn't check out: %s" % haproxy_check.return_code)
|
|
|
|
|
|
def build_haproxy():
|
|
droplets = assign_digitalocean_roledefs(split=True)
|
|
servers = defaultdict(list)
|
|
gunicorn_counts_servers = ["app22", "app26"]
|
|
gunicorn_refresh_servers = ["app20", "app21"]
|
|
maintenance_servers = ["app20"]
|
|
node_socket3_servers = ["node02", "node03"]
|
|
ignore_servers = []
|
|
|
|
for group_type in [
|
|
"app",
|
|
"push",
|
|
"work",
|
|
"node_socket",
|
|
"node_socket3",
|
|
"node_favicon",
|
|
"node_text",
|
|
"www",
|
|
]:
|
|
group_type_name = group_type
|
|
if "node" in group_type:
|
|
group_type_name = "node"
|
|
for server in droplets[group_type_name]:
|
|
droplet_nums = re.findall(r"\d+", server["name"])
|
|
droplet_num = droplet_nums[0] if droplet_nums else ""
|
|
server_type = group_type
|
|
port = 80
|
|
check_inter = 3000
|
|
|
|
if server["name"] in ignore_servers:
|
|
print(" ---> Ignoring %s" % server["name"])
|
|
continue
|
|
if server["name"] in node_socket3_servers and group_type != "node_socket3":
|
|
continue
|
|
if server["name"] not in node_socket3_servers and group_type == "node_socket3":
|
|
continue
|
|
if server_type == "www":
|
|
port = 81
|
|
if group_type == "node_socket":
|
|
port = 8888
|
|
if group_type == "node_socket3":
|
|
port = 8888
|
|
if group_type == "node_text":
|
|
port = 4040
|
|
if group_type in ["app", "push"]:
|
|
port = 8000
|
|
address = "%s:%s" % (server["address"], port)
|
|
|
|
if server_type == "app":
|
|
nginx_address = "%s:80" % (server["address"])
|
|
servers["nginx"].append(
|
|
" server nginx%-15s %-22s check inter 3000ms" % (droplet_num, nginx_address)
|
|
)
|
|
if server["name"] in maintenance_servers:
|
|
nginx_address = "%s:80" % (server["address"])
|
|
servers["maintenance"].append(
|
|
" server nginx%-15s %-22s check inter 3000ms" % (droplet_num, nginx_address)
|
|
)
|
|
|
|
if server["name"] in gunicorn_counts_servers:
|
|
server_type = "gunicorn_counts"
|
|
check_inter = 15000
|
|
elif server["name"] in gunicorn_refresh_servers:
|
|
server_type = "gunicorn_refresh"
|
|
check_inter = 30000
|
|
|
|
server_name = "%s%s" % (server_type, droplet_num)
|
|
servers[server_type].append(
|
|
" server %-20s %-22s check inter %sms" % (server_name, address, check_inter)
|
|
)
|
|
|
|
h = open(os.path.join(env.NEWSBLUR_PATH, "config/haproxy.conf.template"), "r")
|
|
haproxy_template = h.read()
|
|
for sub, server_list in list(servers.items()):
|
|
sorted_servers = "\n".join(sorted(server_list))
|
|
haproxy_template = haproxy_template.replace("{{ %s }}" % sub, sorted_servers)
|
|
f = open(os.path.join(env.SECRETS_PATH, "configs/haproxy.conf"), "w")
|
|
f.write(haproxy_template)
|
|
f.close()
|
|
|
|
|
|
def upgrade_django(role=None):
|
|
if not role:
|
|
role = role_for_host()
|
|
|
|
with virtualenv(), settings(warn_only=True):
|
|
sudo("sudo dpkg --configure -a")
|
|
setup_supervisor()
|
|
pull()
|
|
run("git co django1.11")
|
|
if role == "task":
|
|
sudo("supervisorctl stop celery")
|
|
run("./utils/kill_celery.sh")
|
|
copy_task_settings()
|
|
enable_celery_supervisor(update=False)
|
|
elif role == "work":
|
|
copy_app_settings()
|
|
enable_celerybeat()
|
|
elif role == "web" or role == "app":
|
|
sudo("supervisorctl stop gunicorn")
|
|
run("./utils/kill_gunicorn.sh")
|
|
copy_app_settings()
|
|
setup_gunicorn(restart=False)
|
|
elif role == "node":
|
|
copy_app_settings()
|
|
config_node(full=True)
|
|
else:
|
|
copy_task_settings()
|
|
|
|
pip()
|
|
clean()
|
|
|
|
# sudo('reboot')
|
|
|
|
|
|
def clean():
|
|
with virtualenv(), settings(warn_only=True):
|
|
run('find . -name "*.pyc" -exec rm -f {} \;')
|
|
|
|
|
|
def downgrade_django(role=None):
|
|
with virtualenv(), settings(warn_only=True):
|
|
pull()
|
|
run("git co master")
|
|
pip()
|
|
run("pip uninstall -y django-paypal")
|
|
if role == "task":
|
|
copy_task_settings()
|
|
enable_celery_supervisor()
|
|
else:
|
|
copy_app_settings()
|
|
deploy()
|
|
|
|
|
|
def vendorize_paypal():
|
|
with virtualenv(), settings(warn_only=True):
|
|
run("pip uninstall -y django-paypal")
|
|
|
|
|
|
def upgrade_pil():
|
|
with virtualenv():
|
|
pull()
|
|
run("pip install --upgrade pillow")
|
|
# celery_stop()
|
|
sudo("apt-get remove -y python-imaging")
|
|
sudo("supervisorctl reload")
|
|
# kill()
|
|
|
|
|
|
def downgrade_pil():
|
|
with virtualenv():
|
|
sudo("apt-get install -y python-imaging")
|
|
sudo("rm -fr /usr/local/lib/python2.7/dist-packages/Pillow*")
|
|
pull()
|
|
sudo("supervisorctl reload")
|
|
# kill()
|
|
|
|
|
|
def setup_db_monitor():
|
|
pull()
|
|
with virtualenv():
|
|
sudo("apt-get install -y libpq-dev python2.7-dev")
|
|
run("pip install -r flask/requirements.txt")
|
|
put("flask/supervisor_db_monitor.conf", "/etc/supervisor/conf.d/db_monitor.conf", use_sudo=True)
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl update")
|
|
|
|
|
|
# ==============
|
|
# = Setup - DB =
|
|
# ==============
|
|
|
|
|
|
@parallel
|
|
def setup_db_firewall():
|
|
ports = [
|
|
5432, # PostgreSQL
|
|
27017, # MongoDB
|
|
28017, # MongoDB web
|
|
27019, # MongoDB config
|
|
6379, # Redis
|
|
# 11211, # Memcached
|
|
3060, # Node original page server
|
|
9200, # Elasticsearch
|
|
5000, # DB Monitor
|
|
]
|
|
sudo("ufw --force reset")
|
|
sudo("ufw default deny")
|
|
sudo("ufw allow ssh")
|
|
sudo("ufw allow 80")
|
|
sudo("ufw allow 443")
|
|
|
|
# DigitalOcean
|
|
for ip in set(
|
|
env.roledefs["app"]
|
|
+ env.roledefs["db"]
|
|
+ env.roledefs["debug"]
|
|
+ env.roledefs["task"]
|
|
+ env.roledefs["work"]
|
|
+ env.roledefs["push"]
|
|
+ env.roledefs["www"]
|
|
+ env.roledefs["search"]
|
|
+ env.roledefs["node"]
|
|
):
|
|
sudo("ufw allow proto tcp from %s to any port %s" % (ip, ",".join(map(str, ports))))
|
|
|
|
# EC2
|
|
# for host in set(env.roledefs['ec2task']):
|
|
# ip = re.search('ec2-(\d+-\d+-\d+-\d+)', host).group(1).replace('-', '.')
|
|
# sudo('ufw allow proto tcp from %s to any port %s' % (
|
|
# ip,
|
|
# ','.join(map(str, ports))
|
|
# ))
|
|
|
|
sudo("ufw --force enable")
|
|
|
|
|
|
def setup_rabbitmq():
|
|
sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" | sudo tee -a /etc/apt/sources.list')
|
|
run("wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc")
|
|
sudo("apt-key add rabbitmq-signing-key-public.asc")
|
|
run("rm rabbitmq-signing-key-public.asc")
|
|
sudo("apt-get update")
|
|
sudo("apt-get install -y rabbitmq-server")
|
|
sudo("rabbitmqctl add_user newsblur newsblur")
|
|
sudo("rabbitmqctl add_vhost newsblurvhost")
|
|
sudo('rabbitmqctl set_permissions -p newsblurvhost newsblur ".*" ".*" ".*"')
|
|
|
|
|
|
# def setup_memcached():
|
|
# sudo('apt-get -y install memcached')
|
|
|
|
|
|
def setup_postgres(standby=False):
|
|
shmmax = 17818362112
|
|
hugepages = 9000
|
|
sudo(
|
|
'echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee /etc/apt/sources.list.d/pgdg.list'
|
|
)
|
|
sudo("wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -")
|
|
sudo("apt update")
|
|
sudo("apt install -y postgresql-13")
|
|
put("config/postgresql-13.conf", "/etc/postgresql/13/main/postgresql.conf", use_sudo=True)
|
|
put("config/postgres_hba-13.conf", "/etc/postgresql/13/main/pg_hba.conf", use_sudo=True)
|
|
sudo("mkdir -p /var/lib/postgresql/13/archive")
|
|
sudo("chown -R postgres.postgres /etc/postgresql/13/main")
|
|
sudo("chown -R postgres.postgres /var/lib/postgresql/13/main")
|
|
sudo("chown -R postgres.postgres /var/lib/postgresql/13/archive")
|
|
sudo('echo "%s" | sudo tee /proc/sys/kernel/shmmax' % shmmax)
|
|
sudo('echo "\nkernel.shmmax = %s" | sudo tee -a /etc/sysctl.conf' % shmmax)
|
|
sudo('echo "\nvm.nr_hugepages = %s\n" | sudo tee -a /etc/sysctl.conf' % hugepages)
|
|
run('echo "ulimit -n 100000" > postgresql.defaults')
|
|
sudo("mv postgresql.defaults /etc/default/postgresql")
|
|
sudo("sysctl -p")
|
|
sudo("rm -f /lib/systemd/system/postgresql.service") # Ubuntu 16 has wrong default
|
|
sudo("systemctl daemon-reload")
|
|
sudo("systemctl enable postgresql")
|
|
|
|
if standby:
|
|
put("config/postgresql_recovery.conf", "/var/lib/postgresql/13/recovery.conf", use_sudo=True)
|
|
sudo("chown -R postgres.postgres /var/lib/postgresql/13/recovery.conf")
|
|
|
|
sudo("/etc/init.d/postgresql stop")
|
|
sudo("/etc/init.d/postgresql start")
|
|
|
|
|
|
def config_postgres(standby=False):
|
|
put("config/postgresql-13.conf", "/etc/postgresql/13/main/postgresql.conf", use_sudo=True)
|
|
put("config/postgres_hba.conf", "/etc/postgresql/13/main/pg_hba.conf", use_sudo=True)
|
|
sudo("chown postgres.postgres /etc/postgresql/13/main/postgresql.conf")
|
|
run('echo "ulimit -n 100000" > postgresql.defaults')
|
|
sudo("mv postgresql.defaults /etc/default/postgresql")
|
|
|
|
sudo("/etc/init.d/postgresql reload 13")
|
|
|
|
|
|
def upgrade_postgres():
|
|
sudo(
|
|
'su postgres -c "/usr/lib/postgresql/10/bin/pg_upgrade -b /usr/lib/postgresql/9.4/bin -B /usr/lib/postgresql/10/bin -d /var/lib/postgresql/9.4/main -D /var/lib/postgresql/10/main"'
|
|
)
|
|
|
|
|
|
def copy_postgres_to_standby(master="db01"):
|
|
# http://www.rassoc.com/gregr/weblog/2013/02/16/zero-to-postgresql-streaming-replication-in-10-mins/
|
|
|
|
# Make sure you can ssh from master to slave and back with the postgres user account.
|
|
# Need to give postgres accounts keys in authroized_keys.
|
|
|
|
# local: fab host:new copy_ssh_keys:postgres,private=True
|
|
# new: sudo su postgres; ssh old
|
|
# new: sudo su postgres; ssh db_pgsql
|
|
# old: sudo su postgres; ssh new
|
|
# old: sudo su postgres -c "psql -c \"SELECT pg_start_backup('label', true)\""
|
|
sudo("systemctl stop postgresql")
|
|
sudo("mkdir -p /var/lib/postgresql/9.4/archive")
|
|
sudo("chown postgres.postgres /var/lib/postgresql/9.4/archive")
|
|
with settings(warn_only=True):
|
|
sudo(
|
|
"su postgres -c \"rsync -Pav -e 'ssh -i ~postgres/.ssh/newsblur.key' --stats --progress postgres@%s:/var/lib/postgresql/9.4/main /var/lib/postgresql/9.4/ --exclude postmaster.pid\""
|
|
% master
|
|
)
|
|
put("config/postgresql_recovery.conf", "/var/lib/postgresql/9.4/main/recovery.conf", use_sudo=True)
|
|
sudo("systemctl start postgresql")
|
|
# old: sudo su postgres -c "psql -c \"SELECT pg_stop_backup()\""
|
|
|
|
# Don't forget to add 'setup_postgres_backups' to new
|
|
|
|
|
|
def disable_thp():
|
|
put("config/disable_transparent_hugepages.sh", "/etc/init.d/disable-transparent-hugepages", use_sudo=True)
|
|
sudo("chmod 755 /etc/init.d/disable-transparent-hugepages")
|
|
sudo("update-rc.d disable-transparent-hugepages defaults")
|
|
|
|
|
|
def setup_mongo():
|
|
MONGODB_VERSION = "3.4.24"
|
|
pull()
|
|
disable_thp()
|
|
sudo("systemctl enable rc-local.service") # Enable rc.local
|
|
sudo(
|
|
'echo "#!/bin/sh -e\n\nif test -f /sys/kernel/mm/transparent_hugepage/enabled; then\n\
|
|
echo never > /sys/kernel/mm/transparent_hugepage/enabled\n\
|
|
fi\n\
|
|
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then\n\
|
|
echo never > /sys/kernel/mm/transparent_hugepage/defrag\n\
|
|
fi\n\n\
|
|
exit 0" | sudo tee /etc/rc.local'
|
|
)
|
|
sudo("curl -fsSL https://www.mongodb.org/static/pgp/server-3.4.asc | sudo apt-key add -")
|
|
# sudo('echo "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen" | sudo tee /etc/apt/sources.list.d/mongodb.list')
|
|
# sudo('echo "\ndeb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" | sudo tee -a /etc/apt/sources.list')
|
|
# sudo('echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list')
|
|
sudo(
|
|
'echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.4 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.4.list'
|
|
)
|
|
sudo("apt-get update")
|
|
sudo(
|
|
"apt-get install -y mongodb-org=%s mongodb-org-server=%s mongodb-org-shell=%s mongodb-org-mongos=%s mongodb-org-tools=%s"
|
|
% (MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION, MONGODB_VERSION)
|
|
)
|
|
put(
|
|
"config/mongodb.%s.conf" % ("prod" if env.user != "ubuntu" else "ec2"),
|
|
"/etc/mongodb.conf",
|
|
use_sudo=True,
|
|
)
|
|
put("config/mongodb.service", "/etc/systemd/system/mongodb.service", use_sudo=True)
|
|
run('echo "ulimit -n 100000" > mongodb.defaults')
|
|
sudo("mv mongodb.defaults /etc/default/mongod")
|
|
sudo("mkdir -p /var/log/mongodb")
|
|
sudo("chown mongodb /var/log/mongodb")
|
|
put("config/logrotate.mongo.conf", "/etc/logrotate.d/mongod", use_sudo=True)
|
|
sudo("systemctl enable mongodb")
|
|
|
|
# Reclaim 5% disk space used for root logs. Set to 1%.
|
|
with settings(warn_only=True):
|
|
sudo("tune2fs -m 1 /dev/vda1")
|
|
|
|
|
|
def setup_mongo_configsvr():
|
|
sudo("mkdir -p /var/lib/mongodb_configsvr")
|
|
sudo("chown mongodb.mongodb /var/lib/mongodb_configsvr")
|
|
put("config/mongodb.configsvr.conf", "/etc/mongodb.configsvr.conf", use_sudo=True)
|
|
put("config/mongodb.configsvr-init", "/etc/init.d/mongodb-configsvr", use_sudo=True)
|
|
sudo("chmod u+x /etc/init.d/mongodb-configsvr")
|
|
run('echo "ulimit -n 100000" > mongodb_configsvr.defaults')
|
|
sudo("mv mongodb_configsvr.defaults /etc/default/mongodb_configsvr")
|
|
sudo("update-rc.d -f mongodb-configsvr defaults")
|
|
sudo("/etc/init.d/mongodb-configsvr start")
|
|
|
|
|
|
def setup_mongo_mongos():
|
|
put("config/mongodb.mongos.conf", "/etc/mongodb.mongos.conf", use_sudo=True)
|
|
put("config/mongodb.mongos-init", "/etc/init.d/mongodb-mongos", use_sudo=True)
|
|
sudo("chmod u+x /etc/init.d/mongodb-mongos")
|
|
run('echo "ulimit -n 100000" > mongodb_mongos.defaults')
|
|
sudo("mv mongodb_mongos.defaults /etc/default/mongodb_mongos")
|
|
sudo("update-rc.d -f mongodb-mongos defaults")
|
|
sudo("/etc/init.d/mongodb-mongos restart")
|
|
|
|
|
|
def setup_mongo_mms():
|
|
pull()
|
|
sudo("rm -f /etc/supervisor/conf.d/mongomms.conf")
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl update")
|
|
with cd(env.VENDOR_PATH):
|
|
sudo("apt-get remove -y mongodb-mms-monitoring-agent")
|
|
run(
|
|
"curl -OL https://mms.mongodb.com/download/agent/monitoring/mongodb-mms-monitoring-agent_2.2.0.70-1_amd64.deb"
|
|
)
|
|
sudo("dpkg -i mongodb-mms-monitoring-agent_2.2.0.70-1_amd64.deb")
|
|
run("rm mongodb-mms-monitoring-agent_2.2.0.70-1_amd64.deb")
|
|
put(os.path.join(env.SECRETS_PATH, "settings/mongo_mms_config.txt"), "mongo_mms_config.txt")
|
|
sudo('echo "\n" | sudo tee -a /etc/mongodb-mms/monitoring-agent.config')
|
|
sudo("cat mongo_mms_config.txt | sudo tee -a /etc/mongodb-mms/monitoring-agent.config")
|
|
sudo("start mongodb-mms-monitoring-agent")
|
|
|
|
|
|
def setup_redis(slave=False):
|
|
redis_version = "3.2.6"
|
|
with cd(env.VENDOR_PATH):
|
|
run("wget http://download.redis.io/releases/redis-%s.tar.gz" % redis_version)
|
|
run("tar -xzf redis-%s.tar.gz" % redis_version)
|
|
run("rm redis-%s.tar.gz" % redis_version)
|
|
with cd(os.path.join(env.VENDOR_PATH, "redis-%s" % redis_version)):
|
|
sudo("make install")
|
|
put("config/redis-init", "/etc/init.d/redis", use_sudo=True)
|
|
sudo("chmod u+x /etc/init.d/redis")
|
|
put("config/redis.conf", "/etc/redis.conf", use_sudo=True)
|
|
if slave:
|
|
put("config/redis_slave.conf", "/etc/redis_server.conf", use_sudo=True)
|
|
else:
|
|
put("config/redis_master.conf", "/etc/redis_server.conf", use_sudo=True)
|
|
# sudo('chmod 666 /proc/sys/vm/overcommit_memory', pty=False)
|
|
# run('echo "1" > /proc/sys/vm/overcommit_memory', pty=False)
|
|
# sudo('chmod 644 /proc/sys/vm/overcommit_memory', pty=False)
|
|
disable_thp()
|
|
sudo("systemctl enable rc-local.service") # Enable rc.local
|
|
sudo(
|
|
'echo "#!/bin/sh -e\n\nif test -f /sys/kernel/mm/transparent_hugepage/enabled; then\n\
|
|
echo never > /sys/kernel/mm/transparent_hugepage/enabled\n\
|
|
fi\n\
|
|
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then\n\
|
|
echo never > /sys/kernel/mm/transparent_hugepage/defrag\n\
|
|
fi\n\n\
|
|
exit 0" | sudo tee /etc/rc.local'
|
|
)
|
|
sudo("echo 1 | sudo tee /proc/sys/vm/overcommit_memory")
|
|
sudo('echo "vm.overcommit_memory = 1" | sudo tee -a /etc/sysctl.conf')
|
|
sudo("sysctl vm.overcommit_memory=1")
|
|
put("config/redis_rclocal.txt", "/etc/rc.local", use_sudo=True)
|
|
sudo("chown root.root /etc/rc.local")
|
|
sudo("chmod a+x /etc/rc.local")
|
|
sudo('echo "never" | sudo tee /sys/kernel/mm/transparent_hugepage/enabled')
|
|
run('echo "\nnet.core.somaxconn=65535\n" | sudo tee -a /etc/sysctl.conf', pty=False)
|
|
sudo("mkdir -p /var/lib/redis")
|
|
sudo("update-rc.d redis defaults")
|
|
sudo("/etc/init.d/redis stop")
|
|
sudo("/etc/init.d/redis start")
|
|
setup_syncookies()
|
|
config_monit_redis()
|
|
|
|
|
|
def setup_munin():
|
|
sudo("apt-get update")
|
|
sudo("apt-get install -y munin munin-node munin-plugins-extra spawn-fcgi")
|
|
put("config/munin.conf", "/etc/munin/munin.conf", use_sudo=True) # Only use on main munin
|
|
put("config/spawn_fcgi_munin_graph.conf", "/etc/init.d/spawn_fcgi_munin_graph", use_sudo=True)
|
|
put("config/spawn_fcgi_munin_html.conf", "/etc/init.d/spawn_fcgi_munin_html", use_sudo=True)
|
|
sudo("chmod u+x /etc/init.d/spawn_fcgi_munin_graph")
|
|
sudo("chmod u+x /etc/init.d/spawn_fcgi_munin_html")
|
|
with settings(warn_only=True):
|
|
sudo("chown nginx.www-data /var/log/munin/munin-cgi*")
|
|
sudo("chown nginx.www-data /usr/lib/cgi-bin/munin-cgi*")
|
|
sudo("chown nginx.www-data /usr/lib/munin/cgi/munin-cgi*")
|
|
with settings(warn_only=True):
|
|
sudo("/etc/init.d/spawn_fcgi_munin_graph stop")
|
|
sudo("/etc/init.d/spawn_fcgi_munin_graph start")
|
|
sudo("update-rc.d spawn_fcgi_munin_graph defaults")
|
|
sudo("/etc/init.d/spawn_fcgi_munin_html stop")
|
|
sudo("/etc/init.d/spawn_fcgi_munin_html start")
|
|
sudo("update-rc.d spawn_fcgi_munin_html defaults")
|
|
sudo("/etc/init.d/munin-node stop")
|
|
time.sleep(2)
|
|
sudo("/etc/init.d/munin-node start")
|
|
with settings(warn_only=True):
|
|
sudo("chown nginx.www-data /var/log/munin/munin-cgi*")
|
|
sudo("chown nginx.www-data /usr/lib/cgi-bin/munin-cgi*")
|
|
sudo("chown nginx.www-data /usr/lib/munin/cgi/munin-cgi*")
|
|
sudo("chmod a+rw /var/log/munin/*")
|
|
with settings(warn_only=True):
|
|
sudo("/etc/init.d/spawn_fcgi_munin_graph start")
|
|
sudo("/etc/init.d/spawn_fcgi_munin_html start")
|
|
|
|
|
|
def copy_munin_data(from_server):
|
|
put(os.path.join(env.SECRETS_PATH, "keys/newsblur.key"), "~/.ssh/newsblur.key")
|
|
put(os.path.join(env.SECRETS_PATH, "keys/newsblur.key.pub"), "~/.ssh/newsblur.key.pub")
|
|
run("chmod 600 ~/.ssh/newsblur*")
|
|
|
|
# put("config/munin.nginx.conf", "/usr/local/nginx/conf/sites-enabled/munin.conf", use_sudo=True)
|
|
sudo("/etc/init.d/nginx reload")
|
|
|
|
run(
|
|
'rsync -az -e "ssh -i /home/sclay/.ssh/newsblur.key" --stats --progress %s:/var/lib/munin/ /srv/munin'
|
|
% from_server
|
|
)
|
|
sudo("rm -fr /var/lib/bak-munin")
|
|
sudo("mv /var/lib/munin /var/lib/bak-munin")
|
|
sudo("mv /srv/munin /var/lib/")
|
|
sudo("chown munin.munin -R /var/lib/munin")
|
|
|
|
run(
|
|
'sudo rsync -az -e "ssh -i /home/sclay/.ssh/newsblur.key" --stats --progress %s:/etc/munin/ /srv/munin-etc'
|
|
% from_server
|
|
)
|
|
sudo("rm -fr /etc/munin")
|
|
sudo("mv /srv/munin-etc /etc/munin")
|
|
sudo("chown munin.munin -R /etc/munin")
|
|
|
|
run(
|
|
'sudo rsync -az -e "ssh -i /home/sclay/.ssh/newsblur.key" --stats --progress %s:/var/cache/munin/www/ /srv/munin-www'
|
|
% from_server
|
|
)
|
|
sudo("rm -fr /var/cache/munin/www")
|
|
sudo("mv /srv/munin-www /var/cache/munin/www")
|
|
sudo("chown munin.munin -R /var/cache/munin/www")
|
|
|
|
sudo("/etc/init.d/munin restart")
|
|
sudo("/etc/init.d/munin-node restart")
|
|
|
|
|
|
def setup_db_munin():
|
|
sudo("rm -f /etc/munin/plugins/mongo*")
|
|
sudo("rm -f /etc/munin/plugins/pg_*")
|
|
sudo("rm -f /etc/munin/plugins/redis_*")
|
|
sudo("cp -frs %s/config/munin/mongo* /etc/munin/plugins/" % env.NEWSBLUR_PATH)
|
|
sudo("cp -frs %s/config/munin/pg_* /etc/munin/plugins/" % env.NEWSBLUR_PATH)
|
|
sudo("cp -frs %s/config/munin/redis_* /etc/munin/plugins/" % env.NEWSBLUR_PATH)
|
|
sudo("/etc/init.d/munin-node stop")
|
|
time.sleep(2)
|
|
sudo("/etc/init.d/munin-node start")
|
|
|
|
|
|
def enable_celerybeat():
|
|
with virtualenv():
|
|
run("mkdir -p data")
|
|
put("config/supervisor_celerybeat.conf", "/etc/supervisor/conf.d/celerybeat.conf", use_sudo=True)
|
|
put(
|
|
"config/supervisor_celeryd_work_queue.conf",
|
|
"/etc/supervisor/conf.d/celeryd_work_queue.conf",
|
|
use_sudo=True,
|
|
)
|
|
put("config/supervisor_celeryd_beat.conf", "/etc/supervisor/conf.d/celeryd_beat.conf", use_sudo=True)
|
|
put(
|
|
"config/supervisor_celeryd_beat_feeds.conf",
|
|
"/etc/supervisor/conf.d/celeryd_beat_feeds.conf",
|
|
use_sudo=True,
|
|
)
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl update")
|
|
|
|
|
|
def setup_db_mdadm():
|
|
sudo("apt-get -y install xfsprogs mdadm")
|
|
sudo(
|
|
"yes | mdadm --create /dev/md0 --level=0 -c256 --raid-devices=4 /dev/xvdf /dev/xvdg /dev/xvdh /dev/xvdi"
|
|
)
|
|
sudo("mkfs.xfs /dev/md0")
|
|
sudo("mkdir -p /srv/db")
|
|
sudo("mount -t xfs -o rw,nobarrier,noatime,nodiratime /dev/md0 /srv/db")
|
|
sudo("mkdir -p /srv/db/mongodb")
|
|
sudo("chown mongodb.mongodb /srv/db/mongodb")
|
|
sudo("echo 'DEVICE /dev/xvdf /dev/xvdg /dev/xvdh /dev/xvdi' | sudo tee -a /etc/mdadm/mdadm.conf")
|
|
sudo("mdadm --examine --scan | sudo tee -a /etc/mdadm/mdadm.conf")
|
|
sudo(
|
|
"echo '/dev/md0 /srv/db xfs rw,nobarrier,noatime,nodiratime,noauto 0 0' | sudo tee -a /etc/fstab"
|
|
)
|
|
sudo("sudo update-initramfs -u -v -k `uname -r`")
|
|
|
|
|
|
def setup_original_page_server():
|
|
setup_node_app()
|
|
sudo("mkdir -p /srv/originals")
|
|
sudo(
|
|
"chown %s.%s -R /srv/originals" % (env.user, env.user)
|
|
) # We assume that the group is the same name as the user. It's common on linux
|
|
config_monit_original()
|
|
put("config/supervisor_node_original.conf", "/etc/supervisor/conf.d/node_original.conf", use_sudo=True)
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl reload")
|
|
|
|
|
|
def setup_elasticsearch():
|
|
ES_VERSION = "2.4.4"
|
|
sudo("add-apt-repository -y ppa:openjdk-r/ppa")
|
|
sudo("apt-get update")
|
|
sudo("apt-get install openjdk-7-jre -y")
|
|
|
|
with cd(env.VENDOR_PATH):
|
|
run("mkdir -p elasticsearch-%s" % ES_VERSION)
|
|
with cd(os.path.join(env.VENDOR_PATH, "elasticsearch-%s" % ES_VERSION)):
|
|
# run('wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-%s.deb' % ES_VERSION) # For v5+
|
|
run(
|
|
"wget http://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%s.deb"
|
|
% ES_VERSION
|
|
) # For v1-v2
|
|
sudo("dpkg -i elasticsearch-%s.deb" % ES_VERSION)
|
|
if not files.exists("/usr/share/elasticsearch/plugins/head"):
|
|
sudo("/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head")
|
|
|
|
|
|
def setup_db_search():
|
|
put(
|
|
"config/supervisor_celeryd_search_indexer.conf",
|
|
"/etc/supervisor/conf.d/celeryd_search_indexer.conf",
|
|
use_sudo=True,
|
|
)
|
|
put(
|
|
"config/supervisor_celeryd_search_indexer_tasker.conf",
|
|
"/etc/supervisor/conf.d/celeryd_search_indexer_tasker.conf",
|
|
use_sudo=True,
|
|
)
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl update")
|
|
|
|
|
|
def setup_imageproxy(install_go=False):
|
|
# sudo('apt-get update')
|
|
# sudo('apt-get install -y golang')
|
|
if install_go:
|
|
with cd(env.VENDOR_PATH):
|
|
with settings(warn_only=True):
|
|
run("git clone https://github.com/willnorris/imageproxy.git")
|
|
run("wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz")
|
|
run("tar -xzf go1.13.3.linux-amd64.tar.gz")
|
|
run("rm go1.13.3.linux-amd64.tar.gz")
|
|
sudo("rm /usr/bin/go")
|
|
sudo("ln -s /srv/code/go/bin/go /usr/bin/go")
|
|
with cd(os.path.join(env.VENDOR_PATH, "imageproxy")):
|
|
run("go get willnorris.com/go/imageproxy/cmd/imageproxy")
|
|
put(os.path.join(env.SECRETS_PATH, "settings/imageproxy.key"), "/etc/imageproxy.key", use_sudo=True)
|
|
put(
|
|
os.path.join(env.NEWSBLUR_PATH, "config/supervisor_imageproxy.conf"),
|
|
"/etc/supervisor/conf.d/supervisor_imageproxy.conf",
|
|
use_sudo=True,
|
|
)
|
|
sudo("supervisorctl reread")
|
|
sudo("supervisorctl update")
|
|
sudo("ufw allow 443")
|
|
sudo("ufw allow 80")
|
|
put(
|
|
os.path.join(env.NEWSBLUR_PATH, "config/nginx.imageproxy.conf"),
|
|
"/usr/local/nginx/conf/sites-enabled/imageproxy.conf",
|
|
use_sudo=True,
|
|
)
|
|
sudo("/etc/init.d/nginx restart")
|
|
|
|
|
|
@parallel
|
|
def setup_usage_monitor():
|
|
sudo("ln -fs %s/utils/monitor_disk_usage.py /etc/cron.daily/monitor_disk_usage" % env.NEWSBLUR_PATH)
|
|
sudo("/etc/cron.daily/monitor_disk_usage")
|
|
|
|
|
|
@parallel
|
|
def setup_feeds_fetched_monitor():
|
|
sudo("ln -fs %s/utils/monitor_task_fetches.py /etc/cron.hourly/monitor_task_fetches" % env.NEWSBLUR_PATH)
|
|
sudo("/etc/cron.hourly/monitor_task_fetches")
|
|
|
|
|
|
@parallel
|
|
def setup_newsletter_monitor():
|
|
sudo(
|
|
"ln -fs %s/utils/monitor_newsletter_delivery.py /etc/cron.hourly/monitor_newsletter_delivery"
|
|
% env.NEWSBLUR_PATH
|
|
)
|
|
sudo("/etc/cron.hourly/monitor_newsletter_delivery")
|
|
|
|
|
|
@parallel
|
|
def setup_queue_monitor():
|
|
sudo("ln -fs %s/utils/monitor_work_queue.py /etc/cron.hourly/monitor_work_queue" % env.NEWSBLUR_PATH)
|
|
sudo("/etc/cron.hourly/monitor_work_queue")
|
|
|
|
|
|
@parallel
|
|
def setup_redis_monitor():
|
|
run("sleep 5") # Wait for redis to startup so the log file is there
|
|
sudo("ln -fs %s/utils/monitor_redis_bgsave.py /etc/cron.daily/monitor_redis_bgsave" % env.NEWSBLUR_PATH)
|
|
with settings(warn_only=True):
|
|
sudo("/etc/cron.daily/monitor_redis_bgsave")
|
|
|
|
|
|
# ================
|
|
# = Setup - Task =
|
|
# ================
|
|
|
|
|
|
def setup_task_firewall():
|
|
sudo("ufw default deny")
|
|
sudo("ufw allow ssh")
|
|
sudo("ufw allow 80")
|
|
sudo("ufw --force enable")
|
|
|
|
|
|
def setup_motd(role="app"):
|
|
motd = "/etc/update-motd.d/22-newsblur-motd"
|
|
put("config/motd_%s.txt" % role, motd, use_sudo=True)
|
|
sudo("chown root.root %s" % motd)
|
|
sudo("chmod a+x %s" % motd)
|
|
|
|
|
|
def enable_celery_supervisor(queue=None, update=True):
|
|
if not queue:
|
|
put("config/supervisor_celeryd.conf", "/etc/supervisor/conf.d/celeryd.conf", use_sudo=True)
|
|
else:
|
|
put("config/supervisor_celeryd_%s.conf" % queue, "/etc/supervisor/conf.d/celeryd.conf", use_sudo=True)
|
|
|
|
sudo("supervisorctl reread")
|
|
if update:
|
|
sudo("supervisorctl update")
|
|
|
|
|
|
@parallel
|
|
def copy_db_settings():
|
|
return copy_task_settings()
|
|
|
|
|
|
@parallel
|
|
def copy_task_settings():
|
|
server_hostname = run("hostname")
|
|
# if any([(n in server_hostname) for n in ['task', 'db', 'search', 'node', 'push']]):
|
|
host = server_hostname
|
|
# elif env.host:
|
|
# host = env.host.split('.', 2)[0]
|
|
# else:
|
|
# host = env.host_string.split('.', 2)[0]
|
|
|
|
with settings(warn_only=True):
|
|
run("rm -f %s/local_settings.py" % env.NEWSBLUR_PATH)
|
|
put(
|
|
os.path.join(env.SECRETS_PATH, "settings/task_settings.py"),
|
|
"%s/newsblur/local_settings.py" % env.NEWSBLUR_PATH,
|
|
)
|
|
run(
|
|
'echo "\nSERVER_NAME = \\\\"%s\\\\"" >> %s/newsblur/local_settings.py' % (host, env.NEWSBLUR_PATH)
|
|
)
|
|
|
|
|
|
@parallel
|
|
def copy_spam():
|
|
put(os.path.join(env.SECRETS_PATH, "spam/spam.py"), "%s/apps/social/spam.py" % env.NEWSBLUR_PATH)
|
|
|
|
|
|
# =========================
|
|
# = Setup - Digital Ocean =
|
|
# =========================
|
|
|
|
DO_SIZES = {
|
|
"1": "s-1vcpu-1gb",
|
|
"2": "s-1vcpu-2gb",
|
|
"4": "s-2vcpu-4gb",
|
|
"8": "s-4vcpu-8gb",
|
|
"16": "s-6vcpu-16gb",
|
|
"32": "s-8vcpu-32gb",
|
|
"48": "s-12vcpu-48gb",
|
|
"64": "s-16vcpu-64gb",
|
|
"32c": "c-16",
|
|
}
|
|
|
|
|
|
def setup_do(name, size=1, image=None):
|
|
instance_size = DO_SIZES[str(size)]
|
|
doapi = digitalocean.Manager(token=django_settings.DO_TOKEN_FABRIC)
|
|
# droplets = doapi.get_all_droplets()
|
|
# sizes = dict((s.slug, s.slug) for s in doapi.get_all_sizes())
|
|
ssh_key_ids = [k.id for k in doapi.get_all_sshkeys()]
|
|
if not image:
|
|
image = "ubuntu-20-04-x64"
|
|
else:
|
|
images = dict((s.name, s.id) for s in doapi.get_all_images())
|
|
if image == "task":
|
|
image = images["task-2018-02"]
|
|
elif image == "app":
|
|
image = images["app-2018-02"]
|
|
else:
|
|
images = dict((s.name, s.id) for s in doapi.get_all_images())
|
|
print(images)
|
|
|
|
name = do_name(name)
|
|
env.doname = name
|
|
print("Creating droplet: %s" % name)
|
|
instance = digitalocean.Droplet(
|
|
token=django_settings.DO_TOKEN_FABRIC,
|
|
name=name,
|
|
size_slug=instance_size,
|
|
image=image,
|
|
region="nyc1",
|
|
monitoring=True,
|
|
private_networking=True,
|
|
ssh_keys=ssh_key_ids,
|
|
)
|
|
instance.create()
|
|
time.sleep(2)
|
|
instance = digitalocean.Droplet.get_object(django_settings.DO_TOKEN_FABRIC, instance.id)
|
|
print("Booting droplet: %s / %s (size: %s)" % (instance.name, instance.ip_address, instance_size))
|
|
|
|
i = 0
|
|
while True:
|
|
if instance.status == "active":
|
|
print("...booted: %s" % instance.ip_address)
|
|
time.sleep(5)
|
|
break
|
|
elif instance.status == "new":
|
|
print(".", end=" ")
|
|
sys.stdout.flush()
|
|
instance = digitalocean.Droplet.get_object(django_settings.DO_TOKEN_FABRIC, instance.id)
|
|
i += 1
|
|
time.sleep(i)
|
|
else:
|
|
print("!!! Error: %s" % instance.status)
|
|
return
|
|
|
|
host = instance.ip_address
|
|
env.host_string = host
|
|
time.sleep(20)
|
|
add_user_to_do()
|
|
assign_digitalocean_roledefs()
|
|
|
|
|
|
def do_name(name):
|
|
if re.search(r"[0-9]", name):
|
|
print(" ---> Using %s as hostname" % name)
|
|
return name
|
|
else:
|
|
hosts = do_roledefs(split=False)
|
|
hostnames = [host.name for host in hosts]
|
|
existing_hosts = [hostname for hostname in hostnames if name in hostname]
|
|
for i in range(1, 100):
|
|
try_host = "%s%02d" % (name, i)
|
|
if try_host not in existing_hosts:
|
|
print(
|
|
" ---> %s hosts in %s (%s). %s is unused."
|
|
% (len(existing_hosts), name, ", ".join(existing_hosts), try_host)
|
|
)
|
|
return try_host
|
|
|
|
|
|
def add_user_to_do():
|
|
env.user = "root"
|
|
repo_user = "sclay"
|
|
with settings(warn_only=True):
|
|
run("useradd -m %s" % (repo_user))
|
|
setup_sudoers("%s" % (repo_user))
|
|
run("mkdir -p ~%s/.ssh && chmod 700 ~%s/.ssh" % (repo_user, repo_user))
|
|
run("rm -fr ~%s/.ssh/id_dsa*" % (repo_user))
|
|
run('ssh-keygen -t dsa -f ~%s/.ssh/id_dsa -N ""' % (repo_user))
|
|
run("touch ~%s/.ssh/authorized_keys" % (repo_user))
|
|
copy_ssh_keys()
|
|
run("chown %s.%s -R ~%s/.ssh" % (repo_user, repo_user, repo_user))
|
|
env.user = repo_user
|
|
|
|
|
|
# ===============
|
|
# = Setup - EC2 =
|
|
# ===============
|
|
|
|
|
|
def setup_ec2():
|
|
AMI_NAME = "ami-834cf1ea" # Ubuntu 64-bit 12.04 LTS
|
|
# INSTANCE_TYPE = 'c1.medium'
|
|
INSTANCE_TYPE = "c1.medium"
|
|
conn = EC2Connection(django_settings.AWS_ACCESS_KEY_ID, django_settings.AWS_SECRET_ACCESS_KEY)
|
|
reservation = conn.run_instances(
|
|
AMI_NAME, instance_type=INSTANCE_TYPE, key_name=env.user, security_groups=["db-mongo"]
|
|
)
|
|
instance = reservation.instances[0]
|
|
print("Booting reservation: %s/%s (size: %s)" % (reservation, instance, INSTANCE_TYPE))
|
|
i = 0
|
|
while True:
|
|
if instance.state == "pending":
|
|
print(".", end=" ")
|
|
sys.stdout.flush()
|
|
instance.update()
|
|
i += 1
|
|
time.sleep(i)
|
|
elif instance.state == "running":
|
|
print("...booted: %s" % instance.public_dns_name)
|
|
time.sleep(5)
|
|
break
|
|
else:
|
|
print("!!! Error: %s" % instance.state)
|
|
return
|
|
|
|
host = instance.public_dns_name
|
|
env.host_string = host
|
|
|
|
|
|
# ==========
|
|
# = Deploy =
|
|
# ==========
|
|
|
|
|
|
@parallel
|
|
def pull(master=False):
|
|
with virtualenv():
|
|
run("git pull")
|
|
if master:
|
|
run("git checkout master")
|
|
run("git pull")
|
|
|
|
|
|
def pre_deploy():
|
|
compress_assets(bundle=True)
|
|
|
|
|
|
@serial
|
|
def post_deploy():
|
|
cleanup_assets()
|
|
|
|
|
|
def role_for_host():
|
|
for role, hosts in list(env.roledefs.items()):
|
|
if env.host in hosts:
|
|
return role
|
|
|
|
|
|
@parallel
|
|
def deploy(fast=False, reload=False):
|
|
role = role_for_host()
|
|
if role in ["work", "search", "debug"]:
|
|
deploy_code(copy_assets=False, fast=fast, reload=True)
|
|
else:
|
|
deploy_code(copy_assets=False, fast=fast, reload=reload)
|
|
|
|
|
|
@parallel
|
|
def deploy_web(fast=False):
|
|
role = role_for_host()
|
|
if role in ["work", "search"]:
|
|
deploy_code(copy_assets=True, fast=fast, reload=True)
|
|
else:
|
|
deploy_code(copy_assets=True, fast=fast)
|
|
|
|
|
|
@parallel
|
|
def deploy_rebuild(fast=False):
|
|
deploy_code(copy_assets=True, fast=fast, rebuild=True)
|
|
|
|
|
|
@parallel
|
|
def kill_gunicorn():
|
|
with virtualenv():
|
|
sudo("pkill -9 -u %s -f gunicorn_django" % env.user)
|
|
|
|
|
|
@parallel
|
|
def deploy_code(copy_assets=False, rebuild=False, fast=False, reload=False):
|
|
with virtualenv():
|
|
run("git pull")
|
|
run("mkdir -p static")
|
|
if rebuild:
|
|
run("rm -fr static/*")
|
|
if copy_assets:
|
|
transfer_assets()
|
|
|
|
with virtualenv():
|
|
with settings(warn_only=True):
|
|
if reload:
|
|
sudo("supervisorctl reload")
|
|
elif fast:
|
|
kill_gunicorn()
|
|
else:
|
|
sudo("kill -HUP `cat /srv/newsblur/logs/gunicorn.pid`")
|
|
|
|
|
|
@parallel
|
|
def kill():
|
|
sudo("supervisorctl reload")
|
|
with settings(warn_only=True):
|
|
if env.user == "ubuntu":
|
|
sudo("./utils/kill_gunicorn.sh")
|
|
else:
|
|
run("./utils/kill_gunicorn.sh")
|
|
|
|
|
|
@parallel
|
|
def deploy_node():
|
|
pull()
|
|
with virtualenv():
|
|
run("sudo supervisorctl restart node_unread")
|
|
run("sudo supervisorctl restart node_unread_ssl")
|
|
run("sudo supervisorctl restart node_favicons")
|
|
run("sudo supervisorctl restart node_text")
|
|
|
|
|
|
def gunicorn_restart():
|
|
restart_gunicorn()
|
|
|
|
|
|
def restart_gunicorn():
|
|
with virtualenv(), settings(warn_only=True):
|
|
run("sudo supervisorctl restart gunicorn")
|
|
|
|
|
|
def gunicorn_stop():
|
|
with virtualenv(), settings(warn_only=True):
|
|
run("sudo supervisorctl stop gunicorn")
|
|
|
|
|
|
def staging():
|
|
with cd("~/staging"):
|
|
run("git pull")
|
|
run("kill -HUP `cat logs/gunicorn.pid`")
|
|
run("curl -s http://dev.newsblur.com > /dev/null")
|
|
run("curl -s http://dev.newsblur.com/m/ > /dev/null")
|
|
|
|
|
|
def staging_build():
|
|
with cd("~/staging"):
|
|
run("git pull")
|
|
run("./manage.py migrate")
|
|
run("kill -HUP `cat logs/gunicorn.pid`")
|
|
run("curl -s http://dev.newsblur.com > /dev/null")
|
|
run("curl -s http://dev.newsblur.com/m/ > /dev/null")
|
|
|
|
|
|
@parallel
|
|
def celery():
|
|
celery_slow()
|
|
|
|
|
|
def celery_slow():
|
|
with virtualenv():
|
|
run("git pull")
|
|
celery_stop()
|
|
celery_start()
|
|
|
|
|
|
@parallel
|
|
def celery_fast():
|
|
with virtualenv():
|
|
run("git pull")
|
|
celery_reload()
|
|
|
|
|
|
@parallel
|
|
def celery_stop():
|
|
with virtualenv():
|
|
sudo("supervisorctl stop celery")
|
|
with settings(warn_only=True):
|
|
if env.user == "ubuntu":
|
|
sudo("./utils/kill_celery.sh")
|
|
else:
|
|
run("./utils/kill_celery.sh")
|
|
|
|
|
|
@parallel
|
|
def celery_start():
|
|
with virtualenv():
|
|
run("sudo supervisorctl start celery")
|
|
run("tail logs/newsblur.log")
|
|
|
|
|
|
@parallel
|
|
def celery_reload():
|
|
with virtualenv():
|
|
run("sudo supervisorctl reload celery")
|
|
run("tail logs/newsblur.log")
|
|
|
|
|
|
def kill_celery():
|
|
with virtualenv():
|
|
with settings(warn_only=True):
|
|
if env.user == "ubuntu":
|
|
sudo("./utils/kill_celery.sh")
|
|
else:
|
|
run("./utils/kill_celery.sh")
|
|
|
|
|
|
def compress_assets(bundle=False):
|
|
local("jammit -c newsblur/assets.yml --base-url https://www.newsblur.com --output static")
|
|
local("tar -czf static.tgz static/*")
|
|
|
|
tries_left = 5
|
|
while True:
|
|
try:
|
|
success = False
|
|
with settings(warn_only=True):
|
|
local("PYTHONPATH=/srv/newsblur python utils/backups/s3.py set static.tgz")
|
|
success = True
|
|
if not success:
|
|
raise Exception("Ack!")
|
|
break
|
|
except Exception as e:
|
|
print(" ***> %s. Trying %s more time%s..." % (e, tries_left, "" if tries_left == 1 else "s"))
|
|
tries_left -= 1
|
|
if tries_left <= 0:
|
|
break
|
|
|
|
|
|
def transfer_assets():
|
|
# filename = "deploy_%s.tgz" % env.commit # Easy rollback? Eh, can just upload it again.
|
|
# run('PYTHONPATH=/srv/newsblur python s3.py get deploy_%s.tgz' % filename)
|
|
run("PYTHONPATH=/srv/newsblur python utils/backups/s3.py get static.tgz")
|
|
# run('mv %s static/static.tgz' % filename)
|
|
run("mv static.tgz static/static.tgz")
|
|
run("tar -xzf static/static.tgz")
|
|
run("rm -f static/static.tgz")
|
|
|
|
|
|
def cleanup_assets():
|
|
local("rm -f static.tgz")
|
|
|
|
|
|
# ===========
|
|
# = Backups =
|
|
# ===========
|
|
|
|
|
|
def setup_redis_backups(name=None):
|
|
# crontab for redis backups, name is either none, story, sessions, pubsub
|
|
crontab = (
|
|
"0 4 * * * /srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_redis%s.py"
|
|
% (("_%s" % name) if name else "")
|
|
)
|
|
run('(crontab -l ; echo "%s") | sort - | uniq - | crontab -' % crontab)
|
|
run("crontab -l")
|
|
|
|
|
|
def setup_mongo_backups():
|
|
# crontab for mongo backups
|
|
crontab = "0 4 * * * /srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_mongo.py"
|
|
run('(crontab -l ; echo "%s") | sort - | uniq - | crontab -' % crontab)
|
|
run("crontab -l")
|
|
|
|
|
|
def setup_postgres_backups():
|
|
# crontab for postgres backups
|
|
crontab = """
|
|
0 4 * * * /srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_psql.py
|
|
0 * * * * sudo find /var/lib/postgresql/13/archive -mtime +1 -exec rm {} \;
|
|
0 * * * * sudo find /var/lib/postgresql/13/archive -type f -mmin +180 -delete"""
|
|
|
|
run('(crontab -l ; echo "%s") | sort - | uniq - | crontab -' % crontab)
|
|
run("crontab -l")
|
|
|
|
|
|
def backup_redis(name=None):
|
|
run(
|
|
"/srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_redis%s.py"
|
|
% (("_%s" % name) if name else "")
|
|
)
|
|
|
|
|
|
def backup_mongo():
|
|
run("/srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_mongo.py")
|
|
|
|
|
|
def backup_postgresql():
|
|
run("/srv/newsblur/venv/newsblur3/bin/python /srv/newsblur/utils/backups/backup_psql.py")
|
|
|
|
|
|
# ===============
|
|
# = Calibration =
|
|
# ===============
|
|
|
|
|
|
def sync_time():
|
|
with settings(warn_only=True):
|
|
sudo("/etc/init.d/ntp stop")
|
|
sudo("ntpdate pool.ntp.org")
|
|
sudo("/etc/init.d/ntp start")
|
|
|
|
|
|
def setup_time_calibration():
|
|
sudo("apt-get -y install ntp")
|
|
put("config/ntpdate.cron", "%s/" % env.NEWSBLUR_PATH)
|
|
sudo("chown root.root %s/ntpdate.cron" % env.NEWSBLUR_PATH)
|
|
sudo("chmod 755 %s/ntpdate.cron" % env.NEWSBLUR_PATH)
|
|
sudo("mv %s/ntpdate.cron /etc/cron.hourly/ntpdate" % env.NEWSBLUR_PATH)
|
|
with settings(warn_only=True):
|
|
sudo("/etc/cron.hourly/ntpdate")
|
|
|
|
|
|
# ==============
|
|
# = Tasks - DB =
|
|
# ==============
|
|
|
|
|
|
def restore_postgres(port=5432, download=False):
|
|
with virtualenv():
|
|
backup_date = "2020-12-03-02-51"
|
|
yes = prompt("Dropping and creating NewsBlur PGSQL db. Sure?")
|
|
if yes != "y":
|
|
return
|
|
if download:
|
|
run("mkdir -p postgres")
|
|
run(
|
|
"PYTHONPATH=%s python utils/backups/s3.py get postgres/backup_postgresql_%s.sql.gz"
|
|
% (env.NEWSBLUR_PATH, backup_date)
|
|
)
|
|
# sudo('su postgres -c "createuser -p %s -U newsblur"' % (port,))
|
|
with settings(warn_only=True):
|
|
# May not exist
|
|
run("dropdb newsblur -p %s -U newsblur" % (port,), pty=False)
|
|
run("sudo -u postgres createuser newsblur -s")
|
|
# May already exist
|
|
run("createdb newsblur -p %s -O newsblur -U newsblur" % (port,), pty=False)
|
|
run(
|
|
"pg_restore -U newsblur -p %s --role=newsblur --dbname=newsblur /srv/newsblur/postgres/backup_postgresql_%s.sql.gz"
|
|
% (port, backup_date),
|
|
pty=False,
|
|
)
|
|
|
|
|
|
def restore_mongo(download=False):
|
|
backup_date = "2020-11-11-04-00"
|
|
if download:
|
|
run("PYTHONPATH=/srv/newsblur python utils/backups/s3.py get backup_mongo_%s.tgz" % (backup_date))
|
|
run("tar -xf backup_mongo_%s.tgz" % backup_date)
|
|
run("mongorestore backup_mongo_%s" % backup_date)
|
|
|
|
|
|
# ======
|
|
# = S3 =
|
|
# ======
|
|
|
|
if django_settings:
|
|
try:
|
|
ACCESS_KEY = django_settings.S3_ACCESS_KEY
|
|
SECRET = django_settings.S3_SECRET
|
|
BUCKET_NAME = django_settings.S3_BACKUP_BUCKET # Note that you need to create this bucket first
|
|
except:
|
|
print(" ---> You need to fix django's settings. Enter python and type `import settings`.")
|
|
|
|
|
|
def save_file_in_s3(filename):
|
|
conn = S3Connection(ACCESS_KEY, SECRET)
|
|
bucket = conn.get_bucket(BUCKET_NAME)
|
|
k = Key(bucket)
|
|
k.key = filename
|
|
|
|
k.set_contents_from_filename(filename)
|
|
|
|
|
|
def get_file_from_s3(filename):
|
|
conn = S3Connection(ACCESS_KEY, SECRET)
|
|
bucket = conn.get_bucket(BUCKET_NAME)
|
|
k = Key(bucket)
|
|
k.key = filename
|
|
|
|
k.get_contents_to_filename(filename)
|
|
|
|
|
|
def list_backup_in_s3():
|
|
conn = S3Connection(ACCESS_KEY, SECRET)
|
|
bucket = conn.get_bucket(BUCKET_NAME)
|
|
|
|
for i, key in enumerate(bucket.get_all_keys()):
|
|
print("[%s] %s" % (i, key.name))
|
|
|
|
|
|
def delete_all_backups():
|
|
# FIXME: validate filename exists
|
|
conn = S3Connection(ACCESS_KEY, SECRET)
|
|
bucket = conn.get_bucket(BUCKET_NAME)
|
|
|
|
for i, key in enumerate(bucket.get_all_keys()):
|
|
print("deleting %s" % (key.name))
|
|
key.delete()
|
|
|
|
|
|
def add_revsys_keys():
|
|
put("~/Downloads/revsys-keys.pub", "revsys_keys")
|
|
run("cat revsys_keys >> ~/.ssh/authorized_keys")
|
|
run("rm revsys_keys")
|
|
|
|
|
|
def upgrade_to_virtualenv(role=None):
|
|
if not role:
|
|
print(" ---> You must specify a role!")
|
|
return
|
|
setup_virtualenv()
|
|
if role == "task" or role == "search":
|
|
celery_stop()
|
|
elif role == "app":
|
|
gunicorn_stop()
|
|
elif role == "node":
|
|
run("sudo supervisorctl stop node_unread")
|
|
run("sudo supervisorctl stop node_favicons")
|
|
elif role == "work":
|
|
sudo("/etc/init.d/supervisor stop")
|
|
kill_pgbouncer(bounce=False)
|
|
setup_installs()
|
|
pip()
|
|
if role == "task":
|
|
enable_celery_supervisor(update=False)
|
|
sudo("reboot")
|
|
elif role == "app":
|
|
setup_gunicorn(supervisor=True, restart=False)
|
|
sudo("reboot")
|
|
elif role == "node":
|
|
deploy_node()
|
|
elif role == "search":
|
|
setup_db_search()
|
|
elif role == "work":
|
|
enable_celerybeat()
|
|
sudo("reboot")
|
|
|
|
|
|
def benchmark():
|
|
run("curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash")
|
|
sudo("apt-get install -y sysbench")
|
|
run("sysbench cpu --cpu-max-prime=20000 run")
|
|
run("sysbench fileio --file-total-size=150G prepare")
|
|
run("sysbench fileio --file-total-size=150G --file-test-mode=rndrw --time=300 --max-requests=0 run")
|
|
run("sysbench fileio --file-total-size=150G cleanup")
|