From ba3618cc19ef5c052ee5d10f3acc215c505702fa Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 17 Aug 2022 09:35:53 -0400 Subject: [PATCH 1/5] ansible_ssh_host -> ansible_host --- .../roles/consul-manager/templates/consul_service.json | 2 +- .../roles/elasticsearch/templates/consul_service.json | 2 +- ansible/roles/grafana/templates/consul_service.json | 2 +- .../roles/mongo-exporter/templates/consul_service.json | 4 ++-- .../mongo/templates/consul_service.analytics.json | 2 +- ansible/roles/mongo/templates/consul_service.json | 2 +- .../roles/node-exporter/templates/consul_service.json | 4 ++-- ansible/roles/node/templates/consul_service.json | 8 ++++---- .../postgres-exporter/templates/consul_service.json | 4 ++-- ansible/roles/postgres/templates/consul_service.json | 2 +- ansible/roles/prometheus/templates/consul_service.json | 4 ++-- .../roles/redis-exporter/templates/consul_service.json | 4 ++-- ansible/roles/redis/templates/consul_service.json | 10 +++++----- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/ansible/roles/consul-manager/templates/consul_service.json b/ansible/roles/consul-manager/templates/consul_service.json index a19bf3ea3..779830156 100644 --- a/ansible/roles/consul-manager/templates/consul_service.json +++ b/ansible/roles/consul-manager/templates/consul_service.json @@ -6,7 +6,7 @@ ], "checks": [{ "id": "consul-manager", - "http": "http://{{ ansible_ssh_host }}:8500", + "http": "http://{{ ansible_host }}:8500", "interval": "15s", "failures_before_critical": 4 }], diff --git a/ansible/roles/elasticsearch/templates/consul_service.json b/ansible/roles/elasticsearch/templates/consul_service.json index 24da279d0..3013f1241 100644 --- a/ansible/roles/elasticsearch/templates/consul_service.json +++ b/ansible/roles/elasticsearch/templates/consul_service.json @@ -7,7 +7,7 @@ "port": 9200, "checks": [{ "id": "es-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/elasticsearch", + "http": "http://{{ ansible_host }}:5579/db_check/elasticsearch", "interval": "15s", "failures_before_critical": 4 }] diff --git a/ansible/roles/grafana/templates/consul_service.json b/ansible/roles/grafana/templates/consul_service.json index 7c02bf926..ef32b287d 100644 --- a/ansible/roles/grafana/templates/consul_service.json +++ b/ansible/roles/grafana/templates/consul_service.json @@ -6,7 +6,7 @@ ], "checks": [{ "id": "grafana-ping", - "http": "http://{{ ansible_ssh_host }}:3000/api/health", + "http": "http://{{ ansible_host }}:3000/api/health", "interval": "15s", "failures_before_critical": 4 }], diff --git a/ansible/roles/mongo-exporter/templates/consul_service.json b/ansible/roles/mongo-exporter/templates/consul_service.json index 7ac4bf233..42deb172e 100644 --- a/ansible/roles/mongo-exporter/templates/consul_service.json +++ b/ansible/roles/mongo-exporter/templates/consul_service.json @@ -9,10 +9,10 @@ "checks": [ { "id": "{{inventory_hostname}}-exporter-ping", - "http": "http://{{ ansible_ssh_host }}:9216", + "http": "http://{{ ansible_host }}:9216", "interval": "15s", "failures_before_critical": 4 } ] } -} \ No newline at end of file +} diff --git a/ansible/roles/mongo/templates/consul_service.analytics.json b/ansible/roles/mongo/templates/consul_service.analytics.json index c68614dfa..0a8984c70 100644 --- a/ansible/roles/mongo/templates/consul_service.analytics.json +++ b/ansible/roles/mongo/templates/consul_service.analytics.json @@ -8,7 +8,7 @@ "port": 27017, "checks": [{ "id": "mongo-analytics-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo_analytics?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/mongo_analytics?consul=1", "interval": "15s" }] } diff --git a/ansible/roles/mongo/templates/consul_service.json b/ansible/roles/mongo/templates/consul_service.json index 192d901a0..483906cd6 100644 --- a/ansible/roles/mongo/templates/consul_service.json +++ b/ansible/roles/mongo/templates/consul_service.json @@ -8,7 +8,7 @@ "port": 27017, "checks": [{ "id": "mongo-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/mongo?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/mongo?consul=1", "interval": "15s", "failures_before_critical": 4 }] diff --git a/ansible/roles/node-exporter/templates/consul_service.json b/ansible/roles/node-exporter/templates/consul_service.json index 4043e7e8f..89b544996 100644 --- a/ansible/roles/node-exporter/templates/consul_service.json +++ b/ansible/roles/node-exporter/templates/consul_service.json @@ -9,10 +9,10 @@ "checks": [ { "id": "{{inventory_hostname}}-node-exporter-ping", - "http": "http://{{ ansible_ssh_host }}:9100", + "http": "http://{{ ansible_host }}:9100", "interval": "15s", "failures_before_critical": 4 } ] } -} \ No newline at end of file +} diff --git a/ansible/roles/node/templates/consul_service.json b/ansible/roles/node/templates/consul_service.json index 992f73ef5..ac8981ea1 100644 --- a/ansible/roles/node/templates/consul_service.json +++ b/ansible/roles/node/templates/consul_service.json @@ -13,13 +13,13 @@ "checks": [{ "id": "{{inventory_hostname}}-ping", {% if item.target_host == "node-images" %} - "http": "http://{{ ansible_ssh_host }}:{{ item.port }}/sc,seLJDaKBog3LLEMDe8cjBefMhnVSibO4RA5boZhWcVZ0=/https://samuelclay.com/static/images/2019%20-%20Cuba.jpg", + "http": "http://{{ ansible_host }}:{{ item.port }}/sc,seLJDaKBog3LLEMDe8cjBefMhnVSibO4RA5boZhWcVZ0=/https://samuelclay.com/static/images/2019%20-%20Cuba.jpg", {% elif item.target_host == "node-favicons" %} - "http": "http://{{ ansible_ssh_host }}:{{ item.port }}/rss_feeds/icon/1", + "http": "http://{{ ansible_host }}:{{ item.port }}/rss_feeds/icon/1", {% elif item.target_host == "node-text" %} - "http": "http://{{ ansible_ssh_host }}:{{ item.port }}/rss_feeds/original_text_fetcher?test=1", + "http": "http://{{ ansible_host }}:{{ item.port }}/rss_feeds/original_text_fetcher?test=1", {% elif item.target_host == "node-page" %} - "http": "http://{{ ansible_ssh_host }}:{{ item.port }}/original_page/1?test=1", + "http": "http://{{ ansible_host }}:{{ item.port }}/original_page/1?test=1", {% endif %} "interval": "15s" }] diff --git a/ansible/roles/postgres-exporter/templates/consul_service.json b/ansible/roles/postgres-exporter/templates/consul_service.json index 1e6e87162..b7ba0238e 100644 --- a/ansible/roles/postgres-exporter/templates/consul_service.json +++ b/ansible/roles/postgres-exporter/templates/consul_service.json @@ -9,10 +9,10 @@ "checks": [ { "id": "{{inventory_hostname}}-exporter-ping", - "http": "http://{{ ansible_ssh_host }}:9187", + "http": "http://{{ ansible_host }}:9187", "interval": "15s", "failures_before_critical": 4 } ] } -} \ No newline at end of file +} diff --git a/ansible/roles/postgres/templates/consul_service.json b/ansible/roles/postgres/templates/consul_service.json index 09f6bdd53..fea3b1bf6 100644 --- a/ansible/roles/postgres/templates/consul_service.json +++ b/ansible/roles/postgres/templates/consul_service.json @@ -11,7 +11,7 @@ "port": 5432, "checks": [{ "id": "postgres-ping", - "http": "http://{{ ansible_ssh_host }}:5579/db_check/postgres?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/postgres?consul=1", "interval": "15s", "failures_before_critical": 4 }] diff --git a/ansible/roles/prometheus/templates/consul_service.json b/ansible/roles/prometheus/templates/consul_service.json index ce79f9697..3672bb3bd 100644 --- a/ansible/roles/prometheus/templates/consul_service.json +++ b/ansible/roles/prometheus/templates/consul_service.json @@ -6,10 +6,10 @@ ], "checks": [{ "id": "prometheus-ping", - "http": "http://{{ ansible_ssh_host }}:9090/metrics", + "http": "http://{{ ansible_host }}:9090/metrics", "interval": "15s", "failures_before_critical": 4 }], "port": 9090 } -} \ No newline at end of file +} diff --git a/ansible/roles/redis-exporter/templates/consul_service.json b/ansible/roles/redis-exporter/templates/consul_service.json index 4ec5e3536..3510cc889 100644 --- a/ansible/roles/redis-exporter/templates/consul_service.json +++ b/ansible/roles/redis-exporter/templates/consul_service.json @@ -9,10 +9,10 @@ "checks": [ { "id": "{{ item.redis_target }}-exporter-ping", - "http": "http://{{ ansible_ssh_host }}:{{ item.port }}", + "http": "http://{{ ansible_host }}:{{ item.port }}", "interval": "15s", "failures_before_critical": 4 } ] } -} \ No newline at end of file +} diff --git a/ansible/roles/redis/templates/consul_service.json b/ansible/roles/redis/templates/consul_service.json index 90c33c83f..13be47953 100644 --- a/ansible/roles/redis/templates/consul_service.json +++ b/ansible/roles/redis/templates/consul_service.json @@ -13,15 +13,15 @@ "checks": [{ "id": "{{inventory_hostname}}-ping", {% if inventory_hostname.startswith('db-redis-story') %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_story?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/redis_story?consul=1", {% elif inventory_hostname.startswith('db-redis-user') %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_user?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/redis_user?consul=1", {% elif inventory_hostname.startswith('db-redis-pubsub') %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_pubsub?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/redis_pubsub?consul=1", {% elif inventory_hostname.startswith('db-redis-sessions') %} - "http": "http://{{ ansible_ssh_host }}:5579/db_check/redis_sessions?consul=1", + "http": "http://{{ ansible_host }}:5579/db_check/redis_sessions?consul=1", {% else %} - "http": "http://{{ ansible_ssh_host }}:5000/db_check/redis?consul=1", + "http": "http://{{ ansible_host }}:5000/db_check/redis?consul=1", {% endif %} "interval": "15s", "failures_before_critical": 4 From 77f8cc2324b92214b0457d56620a52a4d7c7ef62 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 17 Aug 2022 09:51:06 -0400 Subject: [PATCH 2/5] Bumping up sentry and node text servers. --- ansible/inventories/digital_ocean.yml | 3 +++ docker/haproxy/haproxy.consul.cfg.j2 | 9 ++++++--- terraform/digitalocean.tf | 3 ++- terraform/variables.tf | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ansible/inventories/digital_ocean.yml b/ansible/inventories/digital_ocean.yml index da18da2f4..7fc18aa9e 100644 --- a/ansible/inventories/digital_ocean.yml +++ b/ansible/inventories/digital_ocean.yml @@ -16,6 +16,9 @@ groups: node: inventory_hostname.startswith('node') node_socket: inventory_hostname.startswith('node-socket') node_images: inventory_hostname.startswith('node-images') + node_text: inventory_hostname.startswith('node-text') + node_page: inventory_hostname.startswith('node-page') + node_favicons: inventory_hostname.startswith('node-favicons') # debugs: inventory_hostname.startswith('debug') diff --git a/docker/haproxy/haproxy.consul.cfg.j2 b/docker/haproxy/haproxy.consul.cfg.j2 index ca370751f..e035b0716 100644 --- a/docker/haproxy/haproxy.consul.cfg.j2 +++ b/docker/haproxy/haproxy.consul.cfg.j2 @@ -63,7 +63,7 @@ frontend public use_backend app_push if { hdr_end(host) -i push.newsblur.com } use_backend node_socket if { path_beg /v3/socket.io/ } - use_backend node_favicon if { path_beg /rss_feeds/icon/ } + use_backend node_favicons if { path_beg /rss_feeds/icon/ } use_backend node_text if { path_beg /rss_feeds/original_text_fetcher } use_backend node_images if { hdr_end(host) -i imageproxy.newsblur.com } use_backend node_images if { hdr_end(host) -i imageproxy2.newsblur.com } @@ -174,7 +174,7 @@ backend node_socket server {{host}} {{host}}.node.nyc1.consul:8008 {% endfor %} -backend node_favicon +backend node_favicons http-check expect rstatus 200|503 option httpchk GET /rss_feeds/icon/1 balance roundrobin @@ -184,7 +184,10 @@ backend node_text http-check expect rstatus 200|503 option httpchk GET /rss_feeds/original_text_fetcher?test=1 balance roundrobin - server-template node-text 1 _node-text._tcp.service.nyc1.consul:8008 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + {% for host in groups.node_text %} + server {{host}} {{host}}.node.nyc1.consul:8008 + {% endfor %} backend node_page http-check expect rstatus 200|503 diff --git a/terraform/digitalocean.tf b/terraform/digitalocean.tf index d9591f54a..2303806d2 100644 --- a/terraform/digitalocean.tf +++ b/terraform/digitalocean.tf @@ -228,8 +228,9 @@ resource "digitalocean_droplet" "discovery" { } resource "digitalocean_droplet" "node-text" { + count = 2 image = var.droplet_os - name = "node-text" + name = contains([0], count.index) ? "node-text" : "node-text${count.index+1}" region = var.droplet_region size = var.droplet_size ssh_keys = [digitalocean_ssh_key.default.fingerprint] diff --git a/terraform/variables.tf b/terraform/variables.tf index 91d71681b..22809850f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -54,7 +54,7 @@ variable "droplet_os" { variable "sentry_droplet_size" { type = string - default = "s-4vcpu-8gb" + default = "s-8vcpu-16gb" } variable "metrics_droplet_size" { From 798238674a291633775f6bd6cabf54dbd2f4aa18 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Wed, 17 Aug 2022 10:04:09 -0400 Subject: [PATCH 3/5] Favicons count++ --- node/favicons.coffee | 2 +- node/favicons.js | 2 +- terraform/digitalocean.tf | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/node/favicons.coffee b/node/favicons.coffee index 90e1b05b8..2a7faac8b 100644 --- a/node/favicons.coffee +++ b/node/favicons.coffee @@ -71,6 +71,6 @@ favicons = (app) => if ENV_DEV or ENV_DOCKER res.redirect '/media/img/icons/nouns/world.svg' else - res.redirect 'https://beta.newsblur.com/media/img/icons/nouns/world.svg' + res.redirect 'https://newsblur.com/media/img/icons/nouns/world.svg' exports.favicons = favicons diff --git a/node/favicons.js b/node/favicons.js index 410c0cdb4..2732b94aa 100644 --- a/node/favicons.js +++ b/node/favicons.js @@ -88,7 +88,7 @@ if (ENV_DEV || ENV_DOCKER) { return res.redirect('/media/img/icons/nouns/world.svg'); } else { - return res.redirect('https://beta.newsblur.com/media/img/icons/nouns/world.svg'); + return res.redirect('https://newsblur.com/media/img/icons/nouns/world.svg'); } } }); diff --git a/terraform/digitalocean.tf b/terraform/digitalocean.tf index 2303806d2..10c96a3ef 100644 --- a/terraform/digitalocean.tf +++ b/terraform/digitalocean.tf @@ -264,8 +264,9 @@ resource "digitalocean_droplet" "node-socket" { } resource "digitalocean_droplet" "node-favicons" { + count = 2 image = var.droplet_os - name = "node-favicons" + name = "node-favicons${count.index+1}" region = var.droplet_region size = var.droplet_size ssh_keys = [digitalocean_ssh_key.default.fingerprint] From dc35fe1a3e5e385e08580a854955000b7544f0fd Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 19 Aug 2022 08:59:34 -0400 Subject: [PATCH 4/5] Fixing date hack for RSS feeds for folders to not turn off until a year after premium archive has launched. --- apps/reader/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/reader/views.py b/apps/reader/views.py index 30147a31b..a5f9b3824 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1159,7 +1159,7 @@ def starred_stories_rss_feed_tag(request, user_id, secret_token, tag_slug): def folder_rss_feed(request, user_id, secret_token, unread_filter, folder_slug): domain = Site.objects.get_current().domain - + date_hack_2023 = (datetime.datetime.now() > datetime.datetime(2023, 7, 1)) try: user = User.objects.get(pk=user_id) except User.DoesNotExist: @@ -1169,7 +1169,7 @@ def folder_rss_feed(request, user_id, secret_token, unread_filter, folder_slug): feed_ids, folder_title = user_sub_folders.feed_ids_under_folder_slug(folder_slug) usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids) - if feed_ids and user.profile.is_archive: + if feed_ids and user.profile.is_archive and date_hack_2023: params = { "user_id": user.pk, "feed_ids": feed_ids, @@ -1268,7 +1268,7 @@ def folder_rss_feed(request, user_id, secret_token, unread_filter, folder_slug): rss.add_item(**story_data) # TODO: Remove below date hack to accomodate users who paid for premium but want folder rss - if not user.profile.is_archive and (datetime.datetime.now() > datetime.datetime(2023, 7, 1)): + if not user.profile.is_archive and date_hack_2023: story_data = { 'title': "You must have a premium archive subscription on NewsBlur to have RSS feeds for folders.", 'link': "https://%s/?next=premium" % domain, From d049dd0da3fae2ce20295c8bc6426b9139875fd7 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Fri, 19 Aug 2022 09:02:46 -0400 Subject: [PATCH 5/5] Fixing date hack for folder rss feeds. --- apps/reader/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/reader/views.py b/apps/reader/views.py index a5f9b3824..b87301644 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1169,7 +1169,7 @@ def folder_rss_feed(request, user_id, secret_token, unread_filter, folder_slug): feed_ids, folder_title = user_sub_folders.feed_ids_under_folder_slug(folder_slug) usersubs = UserSubscription.subs_for_feeds(user.pk, feed_ids=feed_ids) - if feed_ids and user.profile.is_archive and date_hack_2023: + if feed_ids and ((user.profile.is_archive and date_hack_2023) or (not date_hack_2023)): params = { "user_id": user.pk, "feed_ids": feed_ids,