Merge branch 'master' into catalyst

ansible/playbooks/setup_metrics.yml

@@ -8,16 +8,16 @@
     - motd_role: db
 
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    - {role: 'mongo-exporter', tags: 'mongo-exporter'}
-    - {role: 'postgres-exporter', tags: 'postgres-exporter'}
-    - {role: 'redis-exporter', tags: 'redis-exporter'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'prometheus', tags: ['prometheus', 'metrics']}
-    - {role: 'grafana', tags: ['grafana', 'metrics']}
+    # - {role: 'base', tags: 'base'}
+    # - {role: 'ufw', tags: 'ufw'}
+    # - {role: 'docker', tags: 'docker'}
+    # - {role: 'repo', tags: ['repo', 'pull']}
+    # - {role: 'dnsmasq', tags: 'dnsmasq'}
+    # - {role: 'consul', tags: 'consul'}
+    # - {role: 'consul-client', tags: 'consul'}
+    # - {role: 'mongo-exporter', tags: 'mongo-exporter'}
+    - { role: "postgres-exporter", tags: "postgres-exporter" }
+    - { role: "redis-exporter", tags: "redis-exporter" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }
+    - { role: "prometheus", tags: ["prometheus", "metrics"] }
+    - { role: "grafana", tags: ["grafana", "metrics"] }

ansible/roles/mongo-exporter/tasks/main.yml

@@ -1,4 +1,3 @@
-
 - name: Start mongo-exporter container
   become: yes
   docker_container:
@@ -12,9 +11,9 @@
       - name: newsblurnet
     env:
       # MONGODB_URI: 'mongodb://{{ inventory_hostname }}.node.nyc1.consul:27017/admin?'
-      MONGODB_URI: 'mongodb://{{ mongodb_username }}:{{ mongodb_password }}@{{ inventory_hostname }}.node.nyc1.consul:27017/admin?authSource=admin'
+      MONGODB_URI: "mongodb://{{ mongodb_username }}:{{ mongodb_password }}@{{ inventory_hostname }}.node.nyc1.consul:27017/admin?authSource=admin"
     ports:
-    - '9216:9216'
+      - "9216:9216"
 
 - name: Register mongo-exporter in consul
   tags: consul
@@ -24,5 +23,5 @@
     dest: /etc/consul.d/mongo-exporter.json
   notify:
     - reload consul
-- name: Command to register mongo-exporter
-  command: "consul services register /etc/consul.d/mongo-exporter.json"
+# - name: Command to register mongo-exporter
+#   command: "consul services register /etc/consul.d/mongo-exporter.json"

ansible/roles/sentry/tasks/main.yml

@@ -1,14 +1,23 @@
 ---
+- name: Ensure /srv exists and is owned by user
+  become: yes
+  file:
+    path: /srv
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: 0755
+
 - name: Pull sentry self-hosted github
   git:
     repo: https://github.com/getsentry/self-hosted.git
     dest: /srv/sentry/
-    version: master
+    version: 24.4.1
 
 - name: Updating Sentry
   command:
     chdir: /srv/sentry/
-    cmd: ./install.sh
+    cmd: ./install.sh --no-report-self-hosted-issues
 
 - name: docker-compuse up -d
   command:
@@ -24,4 +33,3 @@
   notify:
     - reload consul
   when: disable_consul_services_ie_staging is not defined
-

ansible/roles/sentry/templates/consul_service.json

@@ -1,6 +1,6 @@
 {
     "service": {
-        "name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
+        "name": "{{ inventory_hostname|regex_replace('\-?\d+', '')|regex_replace("hdb-", "db-") }}",
         "id": "{{ inventory_hostname }}",
         "tags": [
             "sentry"

ansible/setup.yml

@@ -26,4 +26,4 @@
 - import_playbook: playbooks/setup_metrics.yml
   when: "'metrics' in inventory_hostname"
 - import_playbook: playbooks/setup_sentry.yml
-  when: "'sentry' in inventory_hostname"
+  when: "'sentry' in inventory_hostname or 'metrics' in inventory_hostname"

apps/api/views.py

@@ -518,6 +518,7 @@ def ip_addresses(request):
     with open('/srv/newsblur/apps/api/ip_addresses.txt', 'r') as f:
         addresses = f.read()
 
-    mail_admins(f"IP Addresses accessed from {request.META['REMOTE_ADDR']} by {request.user}", addresses)
+    if request.user.is_authenticated:
+        mail_admins(f"IP Addresses accessed from {request.META['REMOTE_ADDR']} by {request.user}", addresses)
 
     return HttpResponse(addresses, content_type='text/plain')

apps/notifications/models.py

@@ -306,14 +306,15 @@ class MUserFeedNotification(mongo.Document):
 
         tokens = MUserNotificationTokens.get_tokens_for_user(self.user_id)
         # To update APNS:
+        # 0. Upgrade to latest openssl: brew install openssl
         # 1. Create certificate signing request in Keychain Access
         # 2. Upload to https://developer.apple.com/account/resources/certificates/list
         # 3. Download to secrets/certificates/ios/aps.cer
-        # 4. Open in Keychain Access:
+        # 4. Open in Keychain Access, Under "My Certificates":
         #    - export "Apple Push Service: com.newsblur.NewsBlur" as aps.p12 (or just use aps.cer in #5)
         #    - export private key as aps_key.p12 WITH A PASSPHRASE (removed later)
         # 5. openssl x509 -in aps.cer -inform DER -out aps.pem -outform PEM
-        # 6. openssl pkcs12 -nocerts -out aps_key.pem -in aps_key.p12
+        # 6. openssl pkcs12 -in aps_key.p12 -out aps_key.pem -nodes -legacy
         # 7. openssl rsa -out aps_key.noenc.pem -in aps_key.pem
         # 7. cat aps.pem aps_key.noenc.pem > aps.p12.pem
         # 8. Verify: openssl s_client -connect gateway.push.apple.com:2195 -cert aps.p12.pem

docker/haproxy/haproxy.consul.cfg.j2

@@ -234,7 +234,7 @@ backend db_redis_user
     {% for host in groups.redis_user %}
         server {{host}} {{host}}.node.nyc1.consul:5579
     {% endfor %}
-    server db-redis-secondary hdb-redis-secondary.node.nyc1.consul:5579
+    server hdb-redis-secondary hdb-redis-secondary.node.nyc1.consul:5579
 
 backend db_redis_story
     option httpchk GET /db_check/redis_story

utils/youtube_fetcher.py

@@ -108,7 +108,8 @@ class YoutubeFetcher:
     def extract_username(self, url):
         if "gdata.youtube.com" in url:
             try:
-                username_groups = re.search(r"gdata.youtube.com/feeds/\w+/users/(\w+)/", url)
+                #  Also handle usernames like `user-name`
+                username_groups = re.search(r"gdata.youtube.com/feeds/\w+/users/([^/]+)/", url)
                 if not username_groups:
                     return
                 return username_groups.group(1)
@@ -160,6 +161,7 @@ class YoutubeFetcher:
         return videos
 
     def fetch_channel_videos(self, channel_id):
+        logging.debug(" ***> ~FBFetching YouTube channel: ~SB%s" % channel_id)
         channel_json = requests.get(
             "https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&id=%s&key=%s"
             % (channel_id, settings.YOUTUBE_API_KEY)
@@ -176,6 +178,7 @@ class YoutubeFetcher:
         return self.fetch_playlist_videos(uploads_list_id, title, description)
 
     def fetch_playlist_videos(self, list_id, title=None, description=None):
+        logging.debug(" ***> ~FBFetching YouTube playlist: ~SB%s" % list_id)
         if not title and not description:
             playlist_json = requests.get(
                 "https://www.googleapis.com/youtube/v3/playlists?part=snippet&id=%s&key=%s"
@@ -201,6 +204,7 @@ class YoutubeFetcher:
         return video_ids, title, description
 
     def fetch_user_videos(self, username, username_key="forUsername"):
+        logging.debug(" ***> ~FBFetching YouTube user: ~SB%s" % username)
         channel_json = requests.get(
             "https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&%s=%s&key=%s"
             % (username_key, username, settings.YOUTUBE_API_KEY)