From 3ba68122d746f7bd9655f2c344b7e3a12609de42 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 3 Aug 2015 12:23:49 -0700 Subject: [PATCH] Updating postgres standby commands. --- apps/profile/models.py | 2 +- config/nginx.newsblur.conf | 7 + config/postgres_hba.conf | 106 +++++++ config/postgresql_standby.conf | 529 --------------------------------- fabfile.py | 76 +++-- 5 files changed, 148 insertions(+), 572 deletions(-) create mode 100644 config/postgres_hba.conf delete mode 100644 config/postgresql_standby.conf diff --git a/apps/profile/models.py b/apps/profile/models.py index 4cadd1518..5b7c0d6a0 100644 --- a/apps/profile/models.py +++ b/apps/profile/models.py @@ -396,7 +396,7 @@ class Profile(models.Model): return True @classmethod - def clear_dead(self, days=30, confirm=False): + def clear_dead_spammers(self, days=30, confirm=False): users = User.objects.filter(date_joined__gte=datetime.datetime.now()-datetime.timedelta(days=days)).order_by('-date_joined') usernames = set() diff --git a/config/nginx.newsblur.conf b/config/nginx.newsblur.conf index e5bb29b67..a6bac2aba 100644 --- a/config/nginx.newsblur.conf +++ b/config/nginx.newsblur.conf @@ -40,6 +40,13 @@ server { return 200; } + location /nginx_status { + stub_status on; # activate stub_status module + access_log off; + allow 127.0.0.1; # restrict access to local only + deny all; + } + error_page 502 @down; location @down { root /srv/newsblur/; diff --git a/config/postgres_hba.conf b/config/postgres_hba.conf new file mode 100644 index 000000000..1347745f0 --- /dev/null +++ b/config/postgres_hba.conf @@ -0,0 +1,106 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", +# "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert". Note that +# "password" sends passwords in clear text; "md5" is preferred since +# it sends encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + + + +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +local all postgres peer + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 md5 +# IPv6 local connections: +host all all ::1/128 md5 +# Allow replication connections from localhost, by a user with the +# replication privilege. +#local replication postgres peer +#host replication postgres 127.0.0.1/32 md5 +#host replication postgres ::1/128 md5 + +local all all peer +local all all trust +host all all 0.0.0.0/0 trust +host newsblur newsblur 0.0.0.0/0 trust +host replication all 0.0.0.0/0 trust +host replication postgres 0.0.0.0/0 trust + diff --git a/config/postgresql_standby.conf b/config/postgresql_standby.conf deleted file mode 100644 index 0d6ca0618..000000000 --- a/config/postgresql_standby.conf +++ /dev/null @@ -1,529 +0,0 @@ -# ----------------------------- -# PostgreSQL configuration file -# ----------------------------- -# -# This file consists of lines of the form: -# -# name = value -# -# (The "=" is optional.) Whitespace may be used. Comments are introduced with -# "#" anywhere on a line. The complete list of parameter names and allowed -# values can be found in the PostgreSQL documentation. -# -# The commented-out settings shown in this file represent the default values. -# Re-commenting a setting is NOT sufficient to revert it to the default value; -# you need to reload the server. -# -# This file is read on server startup and when the server receives a SIGHUP -# signal. If you edit the file on a running system, you have to SIGHUP the -# server for the changes to take effect, or use "pg_ctl reload". Some -# parameters, which are marked below, require a server shutdown and restart to -# take effect. -# -# Any parameter can also be given as a command-line option to the server, e.g., -# "postgres -c log_connections=on". Some parameters can be changed at run time -# with the "SET" SQL command. -# -# Memory units: kB = kilobytes Time units: ms = milliseconds -# MB = megabytes s = seconds -# GB = gigabytes min = minutes -# h = hours -# d = days - - -#------------------------------------------------------------------------------ -# FILE LOCATIONS -#------------------------------------------------------------------------------ - -# The default values of these variables are driven from the -D command-line -# option or PGDATA environment variable, represented here as ConfigDir. - -data_directory = '/var/lib/postgresql/9.4/main' # use data in another directory - # (change requires restart) -hba_file = '/etc/postgresql/9.4/main/pg_hba.conf' # host-based authentication file - # (change requires restart) -ident_file = '/etc/postgresql/9.4/main/pg_ident.conf' # ident configuration file - # (change requires restart) - -# If external_pid_file is not explicitly set, no extra PID file is written. -external_pid_file = '/var/run/postgresql/9.4-main.pid' # write an extra PID file - # (change requires restart) - - -#------------------------------------------------------------------------------ -# CONNECTIONS AND AUTHENTICATION -#------------------------------------------------------------------------------ - -# - Connection Settings - - -listen_addresses = '*' # what IP address(es) to listen on; - # comma-separated list of addresses; - # defaults to 'localhost', '*' = all - # (change requires restart) -port = 5432 # (change requires restart) -max_connections = 1000 # (change requires restart) -# Note: Increasing max_connections costs ~400 bytes of shared memory per -# connection slot, plus lock space (see max_locks_per_transaction). -#superuser_reserved_connections = 3 # (change requires restart) -unix_socket_directories = '/var/run/postgresql' # (change requires restart) -#unix_socket_group = '' # (change requires restart) -#unix_socket_permissions = 0777 # begin with 0 to use octal notation - # (change requires restart) -#bonjour = off # advertise server via Bonjour - # (change requires restart) -#bonjour_name = '' # defaults to the computer name - # (change requires restart) - -# - Security and Authentication - - -#authentication_timeout = 1min # 1s-600s -ssl = false # (change requires restart) -#ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH' # allowed SSL ciphers - # (change requires restart) -#ssl_renegotiation_limit = 512MB # amount of data between renegotiations -#password_encryption = on -#db_user_namespace = off - -# Kerberos and GSSAPI -#krb_server_keyfile = '' -#krb_srvname = 'postgres' # (Kerberos only) -#krb_caseins_users = off - -# - TCP Keepalives - -# see "man 7 tcp" for details - -#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; - # 0 selects the system default -#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; - # 0 selects the system default -#tcp_keepalives_count = 0 # TCP_KEEPCNT; - # 0 selects the system default - - -#------------------------------------------------------------------------------ -# RESOURCE USAGE (except WAL) -#------------------------------------------------------------------------------ - -# - Memory - - -shared_buffers = 512MB # min 128kB - # (change requires restart) -#temp_buffers = 8MB # min 800kB -#max_prepared_transactions = 0 # zero disables the feature - # (change requires restart) -# Note: Increasing max_prepared_transactions costs ~600 bytes of shared memory -# per transaction slot, plus lock space (see max_locks_per_transaction). -# It is not advisable to set max_prepared_transactions nonzero unless you -# actively intend to use prepared transactions. -work_mem = 5MB # min 64kB -maintenance_work_mem = 512MB # min 1MB -#max_stack_depth = 2MB # min 100kB - -# - Kernel Resource Usage - - -#max_files_per_process = 1000 # min 25 - # (change requires restart) -#shared_preload_libraries = '' # (change requires restart) - -# - Cost-Based Vacuum Delay - - -#vacuum_cost_delay = 0ms # 0-100 milliseconds -#vacuum_cost_page_hit = 1 # 0-10000 credits -#vacuum_cost_page_miss = 10 # 0-10000 credits -#vacuum_cost_page_dirty = 20 # 0-10000 credits -#vacuum_cost_limit = 200 # 1-10000 credits - -# - Background Writer - - -#bgwriter_delay = 200ms # 10-10000ms between rounds -#bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round -#bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round - -# - Asynchronous Behavior - - -#effective_io_concurrency = 1 # 1-1000. 0 disables prefetching - - -#------------------------------------------------------------------------------ -# WRITE AHEAD LOG -#------------------------------------------------------------------------------ - -# - Settings - - -wal_level = hot_standby # minimal, archive, or hot_standby - # (change requires restart) -#fsync = on # turns forced synchronization on or off -synchronous_commit = off # immediate fsync at commit -#wal_sync_method = fsync # the default is the first option - # supported by the operating system: - # open_datasync - # fdatasync (default on Linux) - # fsync - # fsync_writethrough - # open_sync -#full_page_writes = on # recover from partial page writes -wal_buffers = 16MB # min 32kB - # (change requires restart) -#wal_writer_delay = 200ms # 1-10000 milliseconds - -#commit_delay = 0 # range 0-100000, in microseconds -#commit_siblings = 5 # range 1-1000 - -# - Checkpoints - - -checkpoint_segments = 10 # in logfile segments, min 1, 16MB each -#checkpoint_timeout = 5min # range 30s-1h -#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 -#checkpoint_warning = 30s # 0 disables - -# - Archiving - - -archive_mode = on # allows archiving to be done - # (change requires restart) -archive_command = 'cp -f %p ../archive/%f' - # command to use to archive a logfile segment -#archive_timeout = 0 # force a logfile segment switch after this - # number of seconds; 0 disables - -# - Streaming Replication - - -max_wal_senders = 5 # max number of walsender processes - # (change requires restart) -#wal_sender_delay = 200ms # walsender cycle time, 1-10000 milliseconds -wal_keep_segments = 32 # in logfile segments, 16MB each; 0 disables -#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed - -# - Standby Servers - - -hot_standby = on # "on" allows queries during recovery - # (change requires restart) -#max_standby_archive_delay = 30s # max delay before canceling queries - # when reading WAL from archive; - # -1 allows indefinite delay -#max_standby_streaming_delay = 30s # max delay before canceling queries - # when reading streaming WAL; - # -1 allows indefinite delay - - -#------------------------------------------------------------------------------ -# QUERY TUNING -#------------------------------------------------------------------------------ - -# - Planner Method Configuration - - -#enable_bitmapscan = on -#enable_hashagg = on -#enable_hashjoin = on -#enable_indexscan = on -#enable_material = on -#enable_mergejoin = on -#enable_nestloop = on -#enable_seqscan = on -#enable_sort = on -#enable_tidscan = on - -# - Planner Cost Constants - - -#seq_page_cost = 1.0 # measured on an arbitrary scale -#random_page_cost = 4.0 # same scale as above -#cpu_tuple_cost = 0.01 # same scale as above -#cpu_index_tuple_cost = 0.005 # same scale as above -#cpu_operator_cost = 0.0025 # same scale as above -effective_cache_size = 6GB - -# - Genetic Query Optimizer - - -#geqo = on -#geqo_threshold = 12 -#geqo_effort = 5 # range 1-10 -#geqo_pool_size = 0 # selects default based on effort -#geqo_generations = 0 # selects default based on effort -#geqo_selection_bias = 2.0 # range 1.5-2.0 -#geqo_seed = 0.0 # range 0.0-1.0 - -# - Other Planner Options - - -#default_statistics_target = 100 # range 1-10000 -#constraint_exclusion = partition # on, off, or partition -#cursor_tuple_fraction = 0.1 # range 0.0-1.0 -#from_collapse_limit = 8 -#join_collapse_limit = 8 # 1 disables collapsing of explicit - # JOIN clauses - - -#------------------------------------------------------------------------------ -# ERROR REPORTING AND LOGGING -#------------------------------------------------------------------------------ - -# - Where to Log - - -#log_destination = 'stderr' # Valid values are combinations of - # stderr, csvlog, syslog, and eventlog, - # depending on platform. csvlog - # requires logging_collector to be on. - -# This is used when logging to stderr: -#logging_collector = off # Enable capturing of stderr and csvlog - # into log files. Required to be on for - # csvlogs. - # (change requires restart) - -# These are only used if logging_collector is on: -#log_directory = 'pg_log' # directory where log files are written, - # can be absolute or relative to PGDATA -#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, - # can include strftime() escapes -#log_truncate_on_rotation = off # If on, an existing log file of the - # same name as the new log file will be - # truncated rather than appended to. - # But such truncation only occurs on - # time-driven rotation, not on restarts - # or size-driven rotation. Default is - # off, meaning append to existing files - # in all cases. -#log_rotation_age = 1d # Automatic rotation of logfiles will - # happen after that time. 0 disables. -#log_rotation_size = 10MB # Automatic rotation of logfiles will - # happen after that much log output. - # 0 disables. - -# These are relevant when logging to syslog: -#syslog_facility = 'LOCAL0' -#syslog_ident = 'postgres' - -#silent_mode = off # Run server silently. - # DO NOT USE without syslog or - # logging_collector - # (change requires restart) - - -# - When to Log - - -#client_min_messages = notice # values in order of decreasing detail: - # debug5 - # debug4 - # debug3 - # debug2 - # debug1 - # log - # notice - # warning - # error - -#log_min_messages = warning # values in order of decreasing detail: - # debug5 - # debug4 - # debug3 - # debug2 - # debug1 - # info - # notice - # warning - # error - # log - # fatal - # panic - -#log_min_error_statement = error # values in order of decreasing detail: - # debug5 - # debug4 - # debug3 - # debug2 - # debug1 - # info - # notice - # warning - # error - # log - # fatal - # panic (effectively off) - -#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements - # and their durations, > 0 logs only - # statements running at least this number - # of milliseconds - - -# - What to Log - - -#debug_print_parse = off -#debug_print_rewritten = off -#debug_print_plan = off -#debug_pretty_print = on -#log_checkpoints = off -#log_connections = off -#log_disconnections = off -#log_duration = off -#log_error_verbosity = default # terse, default, or verbose messages -#log_hostname = off -log_line_prefix = '%t %h' # special values: - # %a = application name - # %u = user name - # %d = database name - # %r = remote host and port - # %h = remote host - # %p = process ID - # %t = timestamp without milliseconds - # %m = timestamp with milliseconds - # %i = command tag - # %e = SQL state - # %c = session ID - # %l = session line number - # %s = session start timestamp - # %v = virtual transaction ID - # %x = transaction ID (0 if none) - # %q = stop here in non-session - # processes - # %% = '%' - # e.g. '<%u%%%d> ' -#log_lock_waits = off # log lock waits >= deadlock_timeout -#log_statement = 'none' # none, ddl, mod, all -#log_temp_files = -1 # log temporary files equal or larger - # than the specified size in kilobytes; - # -1 disables, 0 logs all temp files -#log_timezone = unknown # actually, defaults to TZ environment - # setting - - -#------------------------------------------------------------------------------ -# RUNTIME STATISTICS -#------------------------------------------------------------------------------ - -# - Query/Index Statistics Collector - - -#track_activities = on -track_counts = on -#track_functions = none # none, pl, all -#track_activity_query_size = 1024 # (change requires restart) -#update_process_title = on -#stats_temp_directory = 'pg_stat_tmp' - - -# - Statistics Monitoring - - -#log_parser_stats = off -#log_planner_stats = off -#log_executor_stats = off -#log_statement_stats = off - - -#------------------------------------------------------------------------------ -# AUTOVACUUM PARAMETERS -#------------------------------------------------------------------------------ - -autovacuum = on # Enable autovacuum subprocess? 'on' - # requires track_counts to also be on. -#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and - # their durations, > 0 logs only - # actions running at least this number - # of milliseconds. -#autovacuum_max_workers = 3 # max number of autovacuum subprocesses - # (change requires restart) -#autovacuum_naptime = 1min # time between autovacuum runs -#autovacuum_vacuum_threshold = 50 # min number of row updates before - # vacuum -#autovacuum_analyze_threshold = 50 # min number of row updates before - # analyze -#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum -#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze -#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum - # (change requires restart) -#autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for - # autovacuum, in milliseconds; - # -1 means use vacuum_cost_delay -#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for - # autovacuum, -1 means use - # vacuum_cost_limit - - -#------------------------------------------------------------------------------ -# CLIENT CONNECTION DEFAULTS -#------------------------------------------------------------------------------ - -# - Statement Behavior - - -#search_path = '"$user",public' # schema names -#default_tablespace = '' # a tablespace name, '' uses the default -#temp_tablespaces = '' # a list of tablespace names, '' uses - # only default tablespace -#check_function_bodies = on -#default_transaction_isolation = 'read committed' -#default_transaction_read_only = off -#session_replication_role = 'origin' -#statement_timeout = 0 # in milliseconds, 0 is disabled -#vacuum_freeze_min_age = 50000000 -#vacuum_freeze_table_age = 150000000 -#bytea_output = 'hex' # hex, escape -#xmlbinary = 'base64' -#xmloption = 'content' - -# - Locale and Formatting - - -datestyle = 'iso, mdy' -#intervalstyle = 'postgres' -#timezone = unknown # actually, defaults to TZ environment - # setting -#timezone_abbreviations = 'Default' # Select the set of available time zone - # abbreviations. Currently, there are - # Default - # Australia - # India - # You can create your own file in - # share/timezonesets/. -#extra_float_digits = 0 # min -15, max 3 -#client_encoding = sql_ascii # actually, defaults to database - # encoding - -# These settings are initialized by initdb, but they can be changed. -lc_messages = 'en_US.UTF-8' # locale for system error message - # strings -lc_monetary = 'en_US.UTF-8' # locale for monetary formatting -lc_numeric = 'en_US.UTF-8' # locale for number formatting -lc_time = 'en_US.UTF-8' # locale for time formatting - -# default configuration for text search -default_text_search_config = 'pg_catalog.english' - -# - Other Defaults - - -#dynamic_library_path = '$libdir' -#local_preload_libraries = '' - - -#------------------------------------------------------------------------------ -# LOCK MANAGEMENT -#------------------------------------------------------------------------------ - -#deadlock_timeout = 1s -#max_locks_per_transaction = 64 # min 10 - # (change requires restart) -# Note: Each lock table slot uses ~270 bytes of shared memory, and there are -# max_locks_per_transaction * (max_connections + max_prepared_transactions) -# lock table slots. - - -#------------------------------------------------------------------------------ -# VERSION/PLATFORM COMPATIBILITY -#------------------------------------------------------------------------------ - -# - Previous PostgreSQL Versions - - -#array_nulls = on -#backslash_quote = safe_encoding # on, off, or safe_encoding -#default_with_oids = off -#escape_string_warning = on -#lo_compat_privileges = off -#sql_inheritance = on -#standard_conforming_strings = off -#synchronize_seqscans = on - -# - Other Platforms and Clients - - -#transform_null_equals = off - - -#------------------------------------------------------------------------------ -# CUSTOMIZED OPTIONS -#------------------------------------------------------------------------------ - -#custom_variable_classes = '' # list of custom variable class names \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index 24862f467..97987397b 100644 --- a/fabfile.py +++ b/fabfile.py @@ -436,7 +436,7 @@ def setup_python(): # sudo('python setup.py install') with settings(warn_only=True): - sudo('su -c \'echo "import sys; sys.setdefaultencoding(\\\\"utf-8\\\\")" > /usr/lib/python2.7/sitecustomize.py\'') + sudo('echo "import sys; sys.setdefaultencoding(\"utf-8\")" | sudo tee /usr/lib/python2.7/sitecustomize.py') sudo("chmod a+r /usr/local/lib/python2.7/dist-packages/httplib2-0.8-py2.7.egg/EGG-INFO/top_level.txt") sudo("chmod a+r /usr/local/lib/python2.7/dist-packages/python_dateutil-2.1-py2.7.egg/EGG-INFO/top_level.txt") sudo("chmod a+r /usr/local/lib/python2.7/dist-packages/httplib2-0.8-py2.7.egg/httplib2/cacerts.txt") @@ -468,14 +468,14 @@ def setup_supervisor(): @parallel def setup_hosts(): put(os.path.join(env.SECRETS_PATH, 'configs/hosts'), '/etc/hosts', use_sudo=True) - sudo('echo "\n\n127.0.0.1 `hostname`" >> /etc/hosts') + sudo('echo "\n\n127.0.0.1 `hostname`" | sudo tee -a /etc/hosts') def config_pgbouncer(): put('config/pgbouncer.conf', 'pgbouncer.conf') sudo('mv pgbouncer.conf /etc/pgbouncer/pgbouncer.ini') put(os.path.join(env.SECRETS_PATH, 'configs/pgbouncer_auth.conf'), 'userlist.txt') sudo('mv userlist.txt /etc/pgbouncer/userlist.txt') - sudo('echo "START=1" > /etc/default/pgbouncer') + sudo('echo "START=1" | sudo tee /etc/default/pgbouncer') sudo('su postgres -c "/etc/init.d/pgbouncer stop"', pty=False) with settings(warn_only=True): sudo('pkill -9 pgbouncer -e') @@ -492,27 +492,27 @@ def bounce_pgbouncer(): def config_monit_task(): put('config/monit_task.conf', '/etc/monit/conf.d/celery.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def config_monit_node(): put('config/monit_node.conf', '/etc/monit/conf.d/node.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def config_monit_original(): put('config/monit_original.conf', '/etc/monit/conf.d/node_original.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def config_monit_app(): put('config/monit_app.conf', '/etc/monit/conf.d/gunicorn.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def config_monit_work(): put('config/monit_work.conf', '/etc/monit/conf.d/work.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def config_monit_redis(): @@ -521,7 +521,7 @@ def config_monit_redis(): put('config/monit_debug.sh', '/etc/monit/monit_debug.sh', use_sudo=True) sudo('chmod a+x /etc/monit/monit_debug.sh') put('config/monit_redis.conf', '/etc/monit/conf.d/redis.conf', use_sudo=True) - sudo('echo "START=yes" > /etc/default/monit') + sudo('echo "START=yes" | sudo tee /etc/default/monit') sudo('/etc/init.d/monit restart') def setup_mongoengine_repo(): @@ -581,15 +581,11 @@ def setup_ulimit(): run('export FILEMAX=`sysctl -n fs.file-max`', pty=False) sudo('mv /etc/security/limits.conf /etc/security/limits.conf.bak', pty=False) sudo('touch /etc/security/limits.conf', pty=False) - sudo('chmod 666 /etc/security/limits.conf', pty=False) - run('echo "root soft nofile 100000" >> /etc/security/limits.conf', pty=False) - run('echo "root hard nofile 100000" >> /etc/security/limits.conf', pty=False) - run('echo "* soft nofile 100000" >> /etc/security/limits.conf', pty=False) - run('echo "* hard nofile 100090" >> /etc/security/limits.conf', pty=False) - sudo('chmod 644 /etc/security/limits.conf', pty=False) - sudo('chmod 666 /etc/sysctl.conf', pty=False) - run('echo "fs.file-max = 100000" >> /etc/sysctl.conf', pty=False) - sudo('chmod 644 /etc/sysctl.conf', pty=False) + run('echo "root soft nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False) + run('echo "root hard nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False) + run('echo "* soft nofile 100000\n" | sudo tee -a /etc/security/limits.conf', pty=False) + run('echo "* hard nofile 100090\n" | sudo tee -a /etc/security/limits.conf', pty=False) + run('echo "fs.file-max = 100000\n" | sudo tee -a /etc/sysctl.conf', pty=False) sudo('sysctl -p') sudo('ulimit -n 100000') connections.connect(env.host_string) @@ -603,11 +599,11 @@ def setup_ulimit(): # sudo chmod 644 /etc/sysctl.conf def setup_syncookies(): - sudo('echo 1 > /proc/sys/net/ipv4/tcp_syncookies') + sudo('echo 1 | sudo tee /proc/sys/net/ipv4/tcp_syncookies') sudo('sudo /sbin/sysctl -w net.ipv4.tcp_syncookies=1') def setup_sudoers(user=None): - sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user)) + sudo('echo "%s ALL=(ALL) NOPASSWD: ALL\n" | sudo tee -a /etc/sudoers"' % (user or env.user)) def setup_nginx(): NGINX_VERSION = '1.6.2' @@ -745,7 +741,7 @@ def setup_haproxy(debug=False): else: put(os.path.join(env.SECRETS_PATH, 'configs/haproxy.conf'), '/etc/haproxy/haproxy.cfg', use_sudo=True) - sudo('echo "ENABLED=1" > /etc/default/haproxy') + sudo('echo "ENABLED=1" | sudo tee /etc/default/haproxy') cert_path = "%s/config/certificates" % env.NEWSBLUR_PATH run('cat %s/newsblur.com.crt > %s/newsblur.pem' % (cert_path, cert_path)) run('cat %s/newsblur.com.key >> %s/newsblur.pem' % (cert_path, cert_path)) @@ -839,7 +835,7 @@ def setup_db_firewall(): sudo('ufw --force enable') def setup_rabbitmq(): - sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list') + sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" | sudo tee -a /etc/apt/sources.list') run('wget http://www.rabbitmq.com/rabbitmq-signing-key-public.asc') sudo('apt-key add rabbitmq-signing-key-public.asc') run('rm rabbitmq-signing-key-public.asc') @@ -854,16 +850,16 @@ def setup_rabbitmq(): def setup_postgres(standby=False): shmmax = 17672445952 - sudo('su root -c "echo \"deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main\" > /etc/apt/sources.list.d/pgdg.list"') # You might have to run this manually + hugepages = 9000 + sudo('echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list') sudo('wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -') sudo('apt-get update') sudo('apt-get -y install postgresql-9.4 postgresql-client-9.4 postgresql-contrib-9.4 libpq-dev') - put('config/postgresql%s.conf' % ( - ('_standby' if standby else ''), - ), '/etc/postgresql/9.4/main/postgresql.conf', use_sudo=True) - sudo('echo "%s" > /proc/sys/kernel/shmmax' % shmmax) - sudo('echo "\nkernel.shmmax = %s" > /etc/sysctl.conf' % shmmax) - sudo('echo "\nvm.nr_hugepages = 9000\n" > /etc/sysctl.conf' % shmmax) + put('config/postgresql.conf', '/etc/postgresql/9.4/main/postgresql.conf', use_sudo=True) + put('config/postgresql_hba.conf', '/etc/postgresql/9.4/main/pg_hba.conf', use_sudo=True) + sudo('echo "%s" | sudo tee /proc/sys/kernel/shmmax' % shmmax) + sudo('echo "\nkernel.shmmax = %s" | sudo tee -a /etc/sysctl.conf' % shmmax) + sudo('echo "\nvm.nr_hugepages = %s\n" | sudo tee -a /etc/sysctl.conf' % hugepages) sudo('sysctl -p') if standby: @@ -873,9 +869,7 @@ def setup_postgres(standby=False): sudo('/etc/init.d/postgresql start') def config_postgres(standby=False): - put('config/postgresql%s.conf' % ( - ('_standby' if standby else ''), - ), '/etc/postgresql/9.4/main/postgresql.conf', use_sudo=True) + put('config/postgresql.conf', '/etc/postgresql/9.4/main/postgresql.conf', use_sudo=True) sudo('/etc/init.d/postgresql reload 9.4') @@ -889,13 +883,13 @@ def copy_postgres_to_standby(master='db01'): # sudo('su postgres -c \"rsync -a --stats --progress /var/lib/postgresql/9.4/main postgres@%s:/var/lib/postgresql/9.4/ --exclude postmaster.pid\"' % slave, pty=False) # sudo('su postgres -c "psql -c \"SELECT pg_stop_backup()\""', pty=False) - sudo('su postgres pg_basebackup -h %s -D /var/lib/postgresql/9.4/main -v -P -X fetch' % master) - sudo('cp /var/lib/postgresql/9.4/recovery.conf /var/lib/postgresql/9.4/main/') + # sudo('su postgres -c "pg_basebackup -h %s -D /var/lib/postgresql/9.4/main -v -P -X fetch"' % master) + put('config/postgresql_recovery.conf', '/var/lib/postgresql/9.4/main/recovery.conf', use_sudo=True) def setup_mongo(): sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10') # sudo('echo "deb http://downloads.mongodb.org/distros/ubuntu 10.10 10gen" >> /etc/apt/sources.list.d/10gen.list') - sudo('echo "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" >> /etc/apt/sources.list') + sudo('echo "\ndeb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen" | sudo tee -a /etc/apt/sources.list') sudo('apt-get update') sudo('apt-get -y install mongodb-10gen') put('config/mongodb.%s.conf' % ('prod' if env.user != 'ubuntu' else 'ec2'), @@ -941,8 +935,8 @@ def setup_mongo_mms(): run('rm mongodb-mms-monitoring-agent_2.2.0.70-1_amd64.deb') put(os.path.join(env.SECRETS_PATH, 'settings/mongo_mms_config.txt'), 'mongo_mms_config.txt') - sudo("echo \"\n\" >> /etc/mongodb-mms/monitoring-agent.config") - sudo('cat mongo_mms_config.txt >> /etc/mongodb-mms/monitoring-agent.config') + sudo("echo \"\n\" | sudo tee -a /etc/mongodb-mms/monitoring-agent.config") + sudo('cat mongo_mms_config.txt | sudo tee -a /etc/mongodb-mms/monitoring-agent.config') sudo('start mongodb-mms-monitoring-agent') def setup_redis(slave=False): @@ -963,15 +957,13 @@ def setup_redis(slave=False): # sudo('chmod 666 /proc/sys/vm/overcommit_memory', pty=False) # run('echo "1" > /proc/sys/vm/overcommit_memory', pty=False) # sudo('chmod 644 /proc/sys/vm/overcommit_memory', pty=False) - sudo("su root -c \"echo \\\"1\\\" > /proc/sys/vm/overcommit_memory\"") - sudo('chmod 666 /etc/sysctl.conf', pty=False) - run('echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf', pty=False) - sudo('chmod 644 /etc/sysctl.conf', pty=False) + sudo("echo 1 | sudo tee /proc/sys/vm/overcommit_memory\"") + sudo('echo "vm.overcommit_memory = 1" | sudo tee -a /etc/sysctl.conf') sudo("sysctl vm.overcommit_memory=1") put('config/redis_rclocal.txt', '/etc/rc.local', use_sudo=True) sudo("chown root.root /etc/rc.local") sudo("chmod a+x /etc/rc.local") - sudo("su root -c \"echo \\\"never\\\" > /sys/kernel/mm/transparent_hugepage/enabled\"") + sudo('echo "never" | sudo tee /sys/kernel/mm/transparent_hugepage/enabled') sudo('mkdir -p /var/lib/redis') sudo('update-rc.d redis defaults') sudo('/etc/init.d/redis stop')