Merge branch 'master' into dejal

.vscode/settings.json

@@ -16,6 +16,7 @@
         "ansible/playbooks/*/*": true,
         "archive/*": true,
         "logs/*": true,
+        "static/*": true,
         "media/fonts": true,
         "static/*.css": true,
         "static/*.js": true,

Makefile

@@ -19,6 +19,10 @@ rebuild:
 	- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker-compose down
 	- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker-compose up -d
 
+collectstatic: 
+	- rm -fr static
+	- docker run --rm -v $(shell pwd):/srv/newsblur newsblur/newsblur_deploy
+
 #creates newsblur, builds new images, and creates/refreshes SSL keys
 nb: pull
 	- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker-compose down
@@ -99,19 +103,23 @@ pull:
 	- docker pull newsblur/newsblur_monitor
 
 build_web:
-	- docker image build . --platform linux/amd64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
+	- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
 build_node: 
-	- docker image build . --platform linux/amd64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node
+	- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node
 build_monitor: 
-	- docker image build . --platform linux/amd64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor
-build: build_web build_node build_monitor
-push_web: build_web
-	- docker push newsblur/newsblur_python3
-push_node: build_node
-	- docker push newsblur/newsblur_node
-push_monitor: build_monitor
-	- docker push newsblur/newsblur_monitor
-push_images: push_web push_node push_monitor
+	- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor
+build_deploy: 
+	- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy
+build: build_web build_node build_monitor build_deploy
+push_web:
+	- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
+push_node:
+	- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node
+push_monitor:
+	- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor
+push_deploy:
+	- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy
+push_images: push_web push_node push_monitor push_deploy
 push: build push_images
 
 # Tasks
@@ -170,3 +178,7 @@ perf-docker:
 
 clean:
 	- find . -name \*.pyc -delete
+
+
+grafana-dashboards:
+	- python3 utils/grafana_backup.py

ansible.cfg

@@ -7,6 +7,7 @@ private_key_file = /srv/secrets-newsblur/keys/docker.key
 remote_tmp = ~/.ansible/tmp
 forks = 20
 interpreter_python = python3
+stdout_callback = debug
 
 [inventory]
 enable_plugins = ini, constructed

ansible/deploy.yml

@@ -1,6 +1,6 @@
 ---
 - import_playbook: playbooks/deploy_app.yml
-  when: "'app' in group_names"
+  when: "'app' in group_names or 'staging' in group_names"
 - import_playbook: playbooks/deploy_www.yml
   when: "'haproxy' in group_names"
 - import_playbook: playbooks/deploy_node.yml

ansible/env_vars/base.yml

@@ -7,10 +7,16 @@ git_secrets_repo: ssh://git@github.com/samuelclay/newsblur-secrets
 create_user: nb
 local_key: "{{ lookup('file', lookup('env','HOME') + '/.ssh/id_rsa.pub') }}"
 copy_local_key: "{{ lookup('file', '/srv/secrets-newsblur/keys/docker.key.pub') }}"
+postgres_user: "{{ lookup('ini', 'postgres_user section=admin file=/srv/secrets-newsblur/configs/postgres_auth.ini') }}"
 postgres_password: "{{ lookup('ini', 'postgres_password section=admin file=/srv/secrets-newsblur/configs/postgres_auth.ini') }}"
 mongodb_keyfile: "{{ lookup('file', '/srv/secrets-newsblur/keys/mongodb_keyfile.key') }}"
 mongodb_username: "{{ lookup('ini', 'mongodb_username section=admin file=/srv/secrets-newsblur/configs/mongodb_auth.ini') }}"
 mongodb_password: "{{ lookup('ini', 'mongodb_password section=admin file=/srv/secrets-newsblur/configs/mongodb_auth.ini') }}"
+sentry_web_release_webhook: "{{ lookup('ini', 'web_release_webhook section=sentry file=/srv/secrets-newsblur/configs/sentry.ini') }}"
+sentry_task_release_webhook: "{{ lookup('ini', 'task_release_webhook section=sentry file=/srv/secrets-newsblur/configs/sentry.ini') }}"
+sentry_monitor_release_webhook: "{{ lookup('ini', 'monitor_release_webhook section=sentry file=/srv/secrets-newsblur/configs/sentry.ini') }}"
+sentry_node_release_webhook: "{{ lookup('ini', 'node_release_webhook section=sentry file=/srv/secrets-newsblur/configs/sentry.ini') }}"
+
 sys_packages: [ 
   'git', 
   'python3', 

ansible/inventories/digital_ocean.yml

@@ -33,3 +33,4 @@ groups:
   mongo_analytics: inventory_hostname.startswith('db-mongo-analytics')
   consul: inventory_hostname.startswith('db-consul')
   metrics: inventory_hostname.startswith('db-metrics')
+  sentry: inventory_hostname.startswith('db-sentry')

ansible/playbooks/deploy_app.yml

@@ -11,14 +11,39 @@
     #   command: consul leave
     #   ignore_errors: yes
 
-    - name: Compressing JS/CSS assets 
+    - name: Update Sentry release
+      connection: local
+      shell: >
+        curl {{ sentry_web_release_webhook }}/ \
+        -X POST \
+        -H 'Content-Type: application/json' \
+        -d '{"version": "{{ lookup('pipe', 'date "+%Y-%m-%d %H:%M:%S"') }}"}'
+
+    - name: Cleanup static assets before compression
       run_once: yes
       connection: local
-      command: chdir=/srv/newsblur jammit -c /srv/newsblur/newsblur_web/assets.yml --base-url https://www.newsblur.com --output /srv/newsblur/static
+      file:
+        state: absent
+        path: /srv/newsblur/static
+      tags:
+        - never
+        - static
+
+    - name: Updating NewsBlur Deploy container
+      run_once: yes
+      connection: local
+      command: chdir=/srv/newsblur docker pull newsblur/newsblur_deploy
+      tags:
+        - never
+        - static
+
+    - name: Compressing JS/CSS assets 
+      run_once: yes
+      connection: local
+      command: chdir=/srv/newsblur docker run --rm -v /srv/newsblur:/srv/newsblur newsblur/newsblur_deploy
       tags:
         - never
         - static
-        - jammit
     
     - name: Archive JS/CSS assets for uploading
       run_once: yes
@@ -30,6 +55,17 @@
         - never
         - static
       
+    - name: Ensure AWS dependencies installed
+      run_once: yes
+      connection: local
+      pip:
+        name: 
+          - boto3
+          - botocore
+      tags:
+        - never
+        - static
+
     - name: Uploading JS/CSS assets to S3
       run_once: yes
       connection: local
@@ -60,7 +96,7 @@
       amazon.aws.aws_s3:
         bucket: newsblur_backups
         object: /static_py3.tgz
-        dest: /srv/newsblur/static/static.tgz
+        dest: /srv/newsblur/static.tgz
         mode: get
         overwrite: different
         aws_access_key: "{{ lookup('ini', 'aws_access_key_id section=default file=/srv/secrets-newsblur/keys/aws.s3.token') }}"

ansible/playbooks/deploy_monitor.yml

@@ -6,6 +6,14 @@
     - ../env_vars/base.yml
 
   tasks:
+    - name: Update Sentry release
+      connection: local
+      shell: >
+        curl {{ sentry_monitor_release_webhook }}/ \
+        -X POST \
+        -H 'Content-Type: application/json' \
+        -d '{"version": "{{ lookup('pipe', 'date "+%Y-%m-%d %H:%M:%S"') }}"}'
+
     - name: Pull newsblur_web github
       git:
         repo: https://github.com/samuelclay/NewsBlur.git

ansible/playbooks/deploy_node.yml

@@ -6,6 +6,14 @@
     - ../env_vars/base.yml
 
   tasks:
+    - name: Update Sentry release
+      connection: local
+      shell: >
+        curl {{ sentry_node_release_webhook }}/ \
+        -X POST \
+        -H 'Content-Type: application/json' \
+        -d '{"version": "{{ lookup('pipe', 'date "+%Y-%m-%d %H:%M:%S"') }}"}'
+
     - name: Pull newsblur_web github
       git:
         repo: https://github.com/samuelclay/NewsBlur.git

ansible/playbooks/deploy_staging.yml

@@ -17,3 +17,10 @@
       become: yes
       command: "docker kill --signal=HUP newsblur_web"
       # when: pulled.changed
+
+    - name: Update Sentry release
+      shell: >
+        curl {{ sentry_web_release_webhook }}/ \
+        -X POST \
+        -H 'Content-Type: application/json' \
+        -d '{"version": "{{ lookup('pipe', 'date "+%Y-%m-%d %H:%M:%S"') }}"}'

ansible/playbooks/deploy_task.yml

@@ -6,6 +6,14 @@
     - ../env_vars/base.yml
 
   tasks:
+    - name: Update Sentry release
+      connection: local
+      shell: >
+        curl {{ sentry_task_release_webhook }}/ \
+        -X POST \
+        -H 'Content-Type: application/json' \
+        -d '{"version": "{{ lookup('pipe', 'date "+%Y-%m-%d %H:%M:%S"') }}"}'
+
     - name: Pull newsblur_web github
       git:
         repo: https://github.com/samuelclay/NewsBlur.git

ansible/playbooks/setup_app.yml

@@ -1,7 +1,7 @@
 ---
 - name: SETUP -> app containers
   hosts: web
-  serial: "50%"
+  # serial: "50%"
   vars_files:
     - ../env_vars/base.yml
   vars:

ansible/playbooks/setup_sentry.yml

@@ -0,0 +1,20 @@
+---
+- name: SETUP -> sentry containers
+  hosts: sentry
+  vars_files:
+    - ../env_vars/base.yml
+  vars:
+    - update_apt_cache: yes
+    - motd_role: app
+    
+  roles:
+    - {role: 'base', tags: 'base'}
+    - {role: 'ufw', tags: 'ufw'}
+    - {role: 'docker', tags: 'docker'}
+    - {role: 'repo', tags: ['repo', 'pull']}
+    - {role: 'dnsmasq', tags: 'dnsmasq'}
+    - {role: 'consul', tags: 'consul'}
+    - {role: 'consul-client', tags: 'consul'}
+    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
+    
+    - {role: 'sentry', tags: 'sentry'}

ansible/playbooks/setup_www.yml

@@ -18,3 +18,4 @@
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'letsencrypt', tags: 'letsencrypt'}
     - {role: 'haproxy', tags: 'haproxy'}
+    - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics']}

ansible/roles/backups/tasks/main.yml

@@ -17,7 +17,7 @@
   - name: Set backup vars
     set_fact:
       redis_story_filename: backup_redis_story_2021-04-13-04-00.rdb.gz
-      postgres_filename: backup_postgresql_2021-12-17-20-25.sql.gz
+      postgres_filename: backup_postgresql_2022-01-06-19-46.sql.gz
       mongo_filename: backup_mongo_2021-03-15-04-00.tgz
       redis_filename: backup_redis_2021-03-15-04-00.rdb.gz
     tags: never, restore_postgres, restore_mongo, restore_redis, restore_redis_story
@@ -44,16 +44,11 @@
   
   - name: Restore postgres
     block:
-      - name: move postgres archive
-        become: yes
-        command: "mv -f /srv/newsblur/backups/{{ postgres_filename }} /srv/newsblur/docker/volumes/postgres/"
-        ignore_errors: yes
-        
       - name: pg_restore
         become: yes
         command: |
           docker exec -i postgres bash -c 
-          "pg_restore -U newsblur --role=newsblur --dbname=newsblur /var/lib/postgresql/{{ postgres_filename }}"
+          "pg_restore -U newsblur --role=newsblur --dbname=newsblur /var/lib/postgresql/backup/{{ postgres_filename }}"
     tags: never, restore_postgres
   
   - name: Restore mongo

ansible/roles/celery_task/tasks/main.yml

@@ -48,6 +48,7 @@
       timeout: 10s
       retries: 3
       start_period: 30s
+    user: 1000:1001
     volumes:
       - /srv/newsblur:/srv/newsblur
       - /etc/hosts:/etc/hosts

ansible/roles/consul-manager/tasks/main.yml

@@ -26,7 +26,7 @@
   tags: consul
   become: yes
   template:
-    src: /srv/newsblur/ansible/roles/consul-manager/templates/consul_service.json
+    src: consul_service.json
     dest: /etc/consul.d/consul-manager.json
   notify:
-    - reload consul
+    - reload consul

ansible/roles/docker/defaults/main.yml

@@ -22,6 +22,7 @@ docker_prerequisite_packages_Ubuntu:
   - {package: "ca-certificates"}
   - {package: "curl"}
   - {package: "software-properties-common"}
+  - {package: "python3-docker"}
 
 docker_prerequisite_packages_Ubuntu_1404:
   - {package: "linux-image-extra-{{ ansible_kernel }}"}

ansible/roles/docker/tasks/install-Ubuntu.yml

@@ -42,9 +42,35 @@
     state: present
     update_cache: yes
 
-- name: Install Docker Compose
+- name: Check current docker-compose version
+  command: docker-compose --version
+  register: docker_compose_vsn
+  changed_when: false
+  failed_when: false
+  check_mode: no
+  tags: docker-compose
+
+- set_fact:
+    docker_compose_current_version: "{{ docker_compose_vsn.stdout | regex_search('(\\d+(\\.\\d+)+)') }}"
+  when:
+    - docker_compose_vsn.stdout is defined
+  tags: docker-compose
+
+- name: Docker compsoe current version
+  debug: 
+    msg: "{{ docker_compose_current_version }}"
+  tags: docker-compose
+
+- name: Install or upgrade docker-compose
   become: yes
-  apt:
-    name: docker-compose
-    state: present
-    update_cache: yes
+  get_url: 
+    url : "https://github.com/docker/compose/releases/download/v{{ docker_compose_version }}/docker-compose-linux-x86_64"
+    dest: /usr/local/bin/docker-compose
+    mode: 'a+x'
+    force: yes
+  when: >
+    docker_compose_current_version is not defined 
+    or docker_compose_current_version == ""
+    or docker_compose_current_version is version(docker_compose_version, '<')
+  tags: docker-compose
+    

ansible/roles/docker/vars/main.yml

@@ -1,2 +1,2 @@
 ---
-# vars file for docker-ce-ansible-role
+docker_compose_version: 2.2.2

ansible/roles/elasticsearch-exporter/tasks/main.yml

@@ -3,7 +3,7 @@
   tags: consul
   become: yes
   template:
-    src: /srv/newsblur/ansible/roles/elasticsearch-exporter/templates/consul_service.json
+    src: consul_service.json
     dest: /etc/consul.d/elasticsearch_exporter.json
   notify:
     - reload consul

ansible/roles/flask_metrics/tasks/main.yml

@@ -20,7 +20,7 @@
   tags: consul
   become: yes
   template:
-    src: /srv/newsblur/ansible/roles/flask_metrics/templates/consul_service.json
+    src: consul_service.json
     dest: /etc/consul.d/flask_metrics.json
   notify:
     - reload consul
@@ -51,6 +51,8 @@
       file: /srv/newsblur/flask_metrics/flask_metrics_mongo.py
     - service_name: redis
       file: /srv/newsblur/flask_metrics/flask_metrics_redis.py
+    - service_name: www
+      file: /srv/newsblur/flask_metrics/flask_metrics_haproxy.py
 
 - name: Restart flask_metrics
   become: yes

ansible/roles/flask_metrics/templates/consul_service.json

@@ -4,6 +4,8 @@
         "name": "flask_metrics_mongo",
         {% elif 'redis' in inventory_hostname %}
         "name": "flask_metrics_redis",
+        {% elif 'www' in inventory_hostname %}
+        "name": "flask_metrics_haproxy",
         {% endif %}
         "tags": [
             "flask_metrics",

ansible/roles/grafana/tasks/main.yml

@@ -9,7 +9,7 @@
   tags: consul
   become: yes
   template:
-    src: /srv/newsblur/ansible/roles/grafana/templates/consul_service.json
+    src: consul_service.json
     dest: /etc/consul.d/grafana.json
   notify:
     - reload consul
@@ -35,6 +35,10 @@
       - /srv/newsblur/docker/grafana/datasources/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml
       - /srv/newsblur/docker/grafana/dashboards/:/etc/grafana/provisioning/dashboards/
 
+- name: Install sentry pluging
+  shell: >
+    docker exec grafana grafana-cli plugins install grafana-sentry-datasource
+  
 - name: Restart grafana
   debug: 
     msg: Restarting grafana

ansible/roles/nginx/tasks/main.yml

@@ -15,7 +15,7 @@
   become: yes
   docker_container:
     name: nginx
-    image: nginx:1.19
+    image: nginx:1.21
     state: started
     networks_cli_compatible: yes
     network_mode: default

ansible/roles/node/tasks/main.yml

@@ -16,6 +16,8 @@
 - name: Add SERVER_NAME to app secrets
   lineinfile:
     path: /srv/newsblur/node/.env
+    create: yes
+    mode: 0600
     line: 'SERVER_NAME = "{{ inventory_hostname }}"'
 
 - name: Get the volume name
@@ -108,7 +110,7 @@
       - /srv/newsblur/node:/srv/node
   with_items:
     - container_name: imageproxy
-      image: willnorris/imageproxy
+      image: ghcr.io/willnorris/imageproxy
       ports: 8088:8080
       target_host: node-images
   when: item.target_host in inventory_hostname

ansible/roles/postgres/tasks/main.yml

@@ -3,10 +3,17 @@
 - name: Template postgresql-13.conf file
   template:
     src: /srv/newsblur/docker/postgres/postgresql-13.conf.j2
-    dest: /srv/newsblur/docker/postgres/postgresql-13.conf
+    dest: /srv/newsblur/docker/postgres/postgres.conf
   notify: reload postgres
   register: updated_config
 
+- name: Ensure postgres archive directory
+  become: yes
+  file:
+    path: /srv/newsblur/docker/volumes/postgres/archive
+    state: directory
+    mode: 0777
+  
 - name: Start postgres docker containers
   become: yes
   docker_container:
@@ -14,8 +21,9 @@
     image: postgres:13
     state: started
     container_default_behavior: no_defaults
-    command: postgres -D /var/lib/postgresql/pgdata -c config_file=/etc/postgresql/postgresql.conf
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
     env:
+      # POSTGRES_USER: "{{ postgres_user }}" # Don't auto-create newsblur, manually add it
       POSTGRES_PASSWORD: "{{ postgres_password }}"
     hostname: "{{ inventory_hostname }}"
     networks_cli_compatible: yes
@@ -29,11 +37,23 @@
       - 5432:5432
     volumes:
       - /srv/newsblur/docker/volumes/postgres:/var/lib/postgresql
-      - /srv/newsblur/docker/postgres/postgresql-13.conf:/etc/postgresql/postgresql.conf
+      - /srv/newsblur/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf
       - /srv/newsblur/docker/postgres/postgres_hba-13.conf:/etc/postgresql/pg_hba.conf
-      - /backup/:/var/lib/postgresql/backup/
+      - /srv/newsblur/backups/:/var/lib/postgresql/backup/
     restart_policy: unless-stopped
 
+- name: Ensure newsblur role in postgres
+  shell: >
+    sleep 5; 
+    docker exec postgres createuser -s newsblur -U postgres;
+    docker exec postgres createdb newsblur -U newsblur;
+  register: ensure_role
+  changed_when:
+      - "ensure_role.rc == 0"
+  failed_when: 
+    - "'already exists' not in ensure_role.stderr"
+    - "ensure_role.rc != 0"
+
 - name: Register postgres in consul
   tags: consul
   become: yes

ansible/roles/prometheus/tasks/main.yml

@@ -13,7 +13,7 @@
   tags: consul
   become: yes
   template:
-    src: /srv/newsblur/ansible/roles/prometheus/templates/consul_service.json
+    src: consul_service.json
     dest: /etc/consul.d/prometheus.json
   notify:
     - reload consul

ansible/roles/sentry/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+- name: Pull sentry self-hosted github
+  git:
+    repo: https://github.com/getsentry/self-hosted.git
+    dest: /srv/sentry/
+    version: master
+
+- name: Register sentry in consul
+  tags: consul
+  become: yes
+  template:
+    src: consul_service.json
+    dest: /etc/consul.d/sentry.json
+  notify:
+    - reload consul
+  when: disable_consul_services_ie_staging is not defined
+

ansible/roles/sentry/templates/consul_service.json

@@ -0,0 +1,10 @@
+{
+    "service": {
+        "name": "{{ inventory_hostname|regex_replace('\d+', '') }}",
+        "id": "{{ inventory_hostname }}",
+        "tags": [
+            "sentry"
+        ],
+        "port": 9000
+    }
+}  

ansible/roles/web/tasks/main.yml

@@ -29,6 +29,17 @@
   tags:
     - static
 
+- name: Prune docker
+  become: yes
+  community.docker.docker_prune:
+    containers: yes
+    images: yes
+    builder_cache: yes
+    timeout: 300
+  tags: 
+    - prune
+    - never
+
 - name: Install pip
   become: yes
   apt: name=python3-pip state=latest
@@ -66,6 +77,7 @@
     ports:
       - "8000:8000"
     restart_policy: unless-stopped
+    user: 1000:1001
     volumes:
       - /srv/newsblur:/srv/newsblur
       - /etc/hosts:/etc/hosts

ansible/setup.yml

@@ -25,3 +25,5 @@
   when: "'discovery' in inventory_hostname"
 - import_playbook: playbooks/setup_metrics.yml
   when: "'metrics' in inventory_hostname"
+- import_playbook: playbooks/setup_sentry.yml
+  when: "'sentry' in inventory_hostname"

apps/api/views.py

@@ -128,7 +128,7 @@ def add_site(request, token):
     url        = request.GET['url']
     folder     = request.GET['folder']
     new_folder = request.GET.get('new_folder')
-    callback   = request.GET['callback']
+    callback   = request.GET.get('callback', '')
     
     if not url:
         code = -1

apps/newsletters/models.py

@@ -192,6 +192,9 @@ class EmailNewsletter:
             return params['stripped-html']
         if 'body-plain' in params:
             return linkify(linebreaks(params['body-plain']))
+        
+        if force_plain:
+            return self._get_content(params, force_plain=False)
     
     def _clean_content(self, content):
         original = content

apps/push/views.py

@@ -5,6 +5,7 @@ import random
 import datetime
 
 from django.http import HttpResponse, Http404
+from django.http.request import UnreadablePostError
 from django.shortcuts import get_object_or_404
 
 from apps.push.models import PushSubscription
@@ -52,8 +53,11 @@ def push_callback(request, push_id):
         # XXX TODO: Optimize this by removing feedparser. It just needs to find out
         # the hub_url or topic has changed. ElementTree could do it.
         if random.random() < 0.1:
-            parsed = feedparser.parse(request.body)
-            subscription.check_urls_against_pushed_data(parsed)
+            try:
+                parsed = feedparser.parse(request.body)
+                subscription.check_urls_against_pushed_data(parsed)
+            except UnreadablePostError:
+                pass
 
         # Don't give fat ping, just fetch.
         # subscription.feed.queue_pushed_feed_xml(request.body)

apps/reader/views.py

@@ -846,10 +846,9 @@ def load_single_feed(request, feed_id):
     # if not usersub and feed.num_subscribers <= 1:
     #     data = dict(code=-1, message="You must be subscribed to this feed.")
     
+    # time.sleep(random.randint(1, 3))
     if delay and user.is_staff:
-        # import random
         # time.sleep(random.randint(2, 7) / 10.0)
-        # time.sleep(random.randint(1, 10))
         time.sleep(delay)
     # if page == 1:
     #     time.sleep(1)
@@ -2199,7 +2198,11 @@ def delete_feeds_by_folder(request):
 @json.json_view
 def rename_feed(request):
     feed = get_object_or_404(Feed, pk=int(request.POST['feed_id']))
-    user_sub = UserSubscription.objects.get(user=request.user, feed=feed)
+    try:
+        user_sub = UserSubscription.objects.get(user=request.user, feed=feed)
+    except UserSubscription.DoesNotExist:
+        return dict(code=-1, message=f"You are not subscribed to {feed.feed_title}")
+    
     feed_title = request.POST['feed_title']
     
     logging.user(request, "~FRRenaming feed '~SB%s~SN' to: ~SB%s" % (

apps/static/views.py

@@ -29,10 +29,19 @@ def privacy(request):
 
 def tos(request):
     return render(request, 'static/tos.xhtml')
-                        
+
+def webmanifest(request):
+    filename = settings.MEDIA_ROOT + '/extensions/edge/manifest.json'
+    manifest = open(filename).read()
+    
+    return HttpResponse(manifest, content_type='application/manifest+json')
+
 def apple_app_site_assoc(request):
     return render(request, 'static/apple_app_site_assoc.xhtml')
                                                       
+def apple_developer_merchantid(request):
+    return render(request, 'static/apple_developer_merchantid.xhtml')
+
 def feedback(request):
     return render(request, 'static/feedback.xhtml')
 

clients/android/NewsBlur/AndroidManifest.xml

@@ -136,7 +136,7 @@
             android:label="@string/mute_sites"/>
 
         <activity
-            android:name=".activity.SearchForFeeds"
+            android:name=".activity.FeedSearchActivity"
             android:launchMode="singleTop" />
 
         <activity
@@ -147,6 +147,10 @@
             android:permission="android.permission.BIND_JOB_SERVICE" />
         <service android:name=".widget.WidgetRemoteViewsService"
             android:permission="android.permission.BIND_REMOTEVIEWS" />
+        <service
+            android:name=".service.SubscriptionSyncService"
+            android:permission="android.permission.BIND_JOB_SERVICE"
+            android:exported="false" />
 
         <receiver android:name=".service.BootReceiver"
             android:exported="true">

clients/android/NewsBlur/build.gradle

@@ -32,7 +32,7 @@ dependencies {
     implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
     implementation 'com.squareup.okhttp3:okhttp:4.9.2'
     implementation 'com.google.code.gson:gson:2.8.6'
-    implementation 'com.android.billingclient:billing:3.0.3'
+    implementation 'com.android.billingclient:billing:4.0.0'
     implementation 'nl.dionsegijn:konfetti:1.2.2'
     implementation 'com.google.android.play:core:1.10.2'
     implementation "com.google.android.material:material:1.4.0"
@@ -49,8 +49,8 @@ android {
         applicationId "com.newsblur"
         minSdkVersion 21
         targetSdkVersion 31
-        versionCode 198
-        versionName "11.1.1"
+        versionCode 199
+        versionName "11.2"
     }
     compileOptions.with {
         sourceCompatibility = JavaVersion.VERSION_1_8

clients/android/NewsBlur/gradle.properties

@@ -1,2 +1,3 @@
 android.enableJetifier=true
-android.useAndroidX=true
+android.useAndroidX=true
+kotlin.code.style=obsolete

clients/android/NewsBlur/src/com/newsblur/activity/FeedChooser.java

@@ -7,17 +7,17 @@ import android.view.MenuInflater;
 import android.view.MenuItem;
 
 import androidx.annotation.Nullable;
-import androidx.loader.content.Loader;
+import androidx.lifecycle.ViewModelProvider;
 
 import com.newsblur.R;
 import com.newsblur.domain.Feed;
 import com.newsblur.domain.Folder;
 import com.newsblur.util.FeedOrderFilter;
-import com.newsblur.util.FeedUtils;
 import com.newsblur.util.FolderViewFilter;
 import com.newsblur.util.ListOrderFilter;
 import com.newsblur.util.PrefsUtils;
 import com.newsblur.util.WidgetBackground;
+import com.newsblur.viewModel.FeedFolderViewModel;
 import com.newsblur.widget.WidgetUtils;
 
 import java.util.ArrayList;
@@ -33,6 +33,7 @@ abstract public class FeedChooser extends NbActivity {
     protected Map<String, Feed> feedMap = new HashMap<>();
     protected ArrayList<String> folderNames = new ArrayList<>();
     protected ArrayList<ArrayList<Feed>> folderChildren = new ArrayList<>();
+    private FeedFolderViewModel feedFolderViewModel;
 
     abstract void bindLayout();
 
@@ -45,10 +46,11 @@ abstract public class FeedChooser extends NbActivity {
     @Override
     protected void onCreate(@Nullable Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
+        feedFolderViewModel = new ViewModelProvider(this).get(FeedFolderViewModel.class);
         bindLayout();
         setupList();
-        loadFeeds();
-        loadFolders();
+        setupObservers();
+        loadData();
     }
 
     @Override
@@ -144,6 +146,11 @@ abstract public class FeedChooser extends NbActivity {
         adapter.setData(this.folderNames, this.folderChildren, this.feeds);
     }
 
+    private void setupObservers() {
+        feedFolderViewModel.getFoldersLiveData().observe(this, this::processFolders);
+        feedFolderViewModel.getFeedsLiveData().observe(this, this::processFeeds);
+    }
+
     private void replaceFeedOrderFilter(FeedOrderFilter feedOrderFilter) {
         PrefsUtils.setFeedChooserFeedOrder(this, feedOrderFilter);
         adapter.replaceFeedOrder(feedOrderFilter);
@@ -165,16 +172,8 @@ abstract public class FeedChooser extends NbActivity {
         WidgetUtils.updateWidget(this);
     }
 
-    private void loadFeeds() {
-        Loader<Cursor> loader = FeedUtils.dbHelper.getFeedsLoader();
-        loader.registerListener(loader.getId(), (loader1, cursor) -> processFeeds(cursor));
-        loader.startLoading();
-    }
-
-    private void loadFolders() {
-        Loader<Cursor> loader = FeedUtils.dbHelper.getFoldersLoader();
-        loader.registerListener(loader.getId(), (loader1, cursor) -> processFolders(cursor));
-        loader.startLoading();
+    private void loadData() {
+        feedFolderViewModel.getData();
     }
 
     private void processFolders(Cursor cursor) {

clients/android/NewsBlur/src/com/newsblur/activity/FeedSearchActivity.kt

@@ -19,7 +19,7 @@ import java.net.MalformedURLException
 import java.net.URL
 import java.util.*
 
-class SearchForFeeds : NbActivity(), OnFeedSearchResultClickListener, AddFeedProgressListener {
+class FeedSearchActivity : NbActivity(), OnFeedSearchResultClickListener, AddFeedProgressListener {
 
     private val supportedUrlProtocols: MutableSet<String> = HashSet(2)
 
@@ -40,7 +40,6 @@ class SearchForFeeds : NbActivity(), OnFeedSearchResultClickListener, AddFeedPro
         setupListeners()
         apiManager = APIManager(this)
         binding.inputSearchQuery.requestFocus()
-        lifecycleScope
     }
 
     override fun onFeedSearchResultClickListener(result: FeedResult) {

clients/android/NewsBlur/src/com/newsblur/activity/FeedSearchAdapter.kt

@@ -0,0 +1,94 @@
+package com.newsblur.activity
+
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import androidx.recyclerview.widget.DiffUtil
+import androidx.recyclerview.widget.RecyclerView
+import com.newsblur.R
+import com.newsblur.databinding.ViewFeedSearchRowBinding
+import com.newsblur.domain.FeedResult
+import com.newsblur.util.FeedUtils
+
+class FeedSearchAdapter(
+        private val onClickListener: OnFeedSearchResultClickListener
+) : RecyclerView.Adapter<FeedSearchAdapter.ViewHolder>() {
+
+    private val resultsList: MutableList<FeedResult> = mutableListOf()
+
+    override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder {
+        val view = LayoutInflater.from(parent.context).inflate(R.layout.view_feed_search_row, parent, false)
+        return ViewHolder(view)
+    }
+
+    override fun onBindViewHolder(holder: ViewHolder, position: Int) {
+        val result = resultsList[position]
+        holder.bind(result)
+    }
+
+    override fun getItemCount(): Int = resultsList.size
+
+    fun replaceAll(results: Array<FeedResult>) {
+        val newResultsList: List<FeedResult> = results.toList()
+        val diffCallback = ResultDiffCallback(resultsList, newResultsList)
+        val diffResult = DiffUtil.calculateDiff(diffCallback)
+        resultsList.clear()
+        resultsList.addAll(newResultsList)
+        diffResult.dispatchUpdatesTo(this)
+    }
+
+    inner class ViewHolder(itemView: View) : RecyclerView.ViewHolder(itemView) {
+
+        private val binding: ViewFeedSearchRowBinding = ViewFeedSearchRowBinding.bind(itemView)
+
+        fun bind(result: FeedResult) {
+            val resultFaviconUrl = result.faviconUrl
+            if (resultFaviconUrl.isNotEmpty()) {
+                FeedUtils.iconLoader?.displayImage(resultFaviconUrl, binding.imgFeedIcon)
+            }
+
+            binding.textTitle.text = result.label
+            binding.textTagline.text = result.tagline
+            val subscribersCountText = binding.root.context.getString(R.string.feed_subscribers, result.numberOfSubscriber)
+            binding.textSubscriptionCount.text = subscribersCountText
+
+            if (result.url.isNotEmpty()) {
+                binding.rowResultAddress.text = result.url
+                binding.rowResultAddress.visibility = View.VISIBLE
+            } else {
+                binding.rowResultAddress.visibility = View.GONE
+            }
+
+            itemView.setOnClickListener {
+                onClickListener.onFeedSearchResultClickListener(result)
+            }
+        }
+    }
+
+    class ResultDiffCallback(
+            private val oldList: List<FeedResult>,
+            private val newList: List<FeedResult>) : DiffUtil.Callback() {
+
+        override fun areItemsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
+            val oldFeedResult = oldList[oldItemPosition]
+            val newFeedResult = newList[newItemPosition]
+            return oldFeedResult == newFeedResult
+        }
+
+        override fun areContentsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
+            val oldFeedResult = oldList[oldItemPosition]
+            val newFeedResult = newList[newItemPosition]
+            return oldFeedResult.id == newFeedResult.id
+                    && oldFeedResult.label == newFeedResult.label
+        }
+
+        override fun getOldListSize(): Int = oldList.size
+
+        override fun getNewListSize(): Int = newList.size
+    }
+
+    interface OnFeedSearchResultClickListener {
+
+        fun onFeedSearchResultClickListener(result: FeedResult)
+    }
+}

clients/android/NewsBlur/src/com/newsblur/activity/FeedSearchResultAdapter.kt

@@ -1,94 +0,0 @@
-package com.newsblur.activity
-
-import android.graphics.Bitmap
-import android.graphics.BitmapFactory
-import android.text.TextUtils
-import android.util.Base64
-import android.view.LayoutInflater
-import android.view.View
-import android.view.ViewGroup
-import androidx.recyclerview.widget.DiffUtil
-import androidx.recyclerview.widget.RecyclerView
-import com.newsblur.R
-import com.newsblur.databinding.ViewFeedSearchRowBinding
-import com.newsblur.domain.FeedResult
-
-internal class FeedSearchAdapter(private val onClickListener: OnFeedSearchResultClickListener) : RecyclerView.Adapter<FeedSearchAdapter.ViewHolder>() {
-
-    private val resultsList: MutableList<FeedResult> = ArrayList()
-
-    override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder {
-        val view = LayoutInflater.from(parent.context).inflate(R.layout.view_feed_search_row, parent, false)
-        return ViewHolder(view)
-    }
-
-    override fun onBindViewHolder(holder: ViewHolder, position: Int) {
-        val result = resultsList[position]
-        var bitmap: Bitmap? = null
-        if (!TextUtils.isEmpty(result.favicon)) {
-            val data = Base64.decode(result.favicon, Base64.DEFAULT)
-            bitmap = BitmapFactory.decodeByteArray(data, 0, data.size)
-        }
-        bitmap?.let {
-            holder.binding.imgFeedIcon.setImageBitmap(bitmap)
-        }
-
-        holder.binding.textTitle.text = result.label
-        holder.binding.textTagline.text = result.tagline
-        val subscribersCountText = holder.binding.root.context.resources.getString(R.string.feed_subscribers, result.numberOfSubscriber)
-        holder.binding.textSubscriptionCount.text = subscribersCountText
-
-        if (!TextUtils.isEmpty(result.url)) {
-            holder.binding.rowResultAddress.text = result.url
-            holder.binding.rowResultAddress.visibility = View.VISIBLE
-        } else {
-            holder.binding.rowResultAddress.visibility = View.GONE
-        }
-
-        holder.itemView.setOnClickListener {
-            onClickListener.onFeedSearchResultClickListener(result)
-        }
-    }
-
-    override fun getItemCount(): Int = resultsList.size
-
-    fun replaceAll(results: Array<FeedResult>) {
-        val newResultsList: List<FeedResult> = results.toList()
-        val diffCallback = ResultDiffCallback(resultsList, newResultsList)
-        val diffResult = DiffUtil.calculateDiff(diffCallback)
-        resultsList.clear()
-        resultsList.addAll(newResultsList)
-        diffResult.dispatchUpdatesTo(this)
-    }
-
-    internal class ViewHolder(itemView: View) : RecyclerView.ViewHolder(itemView) {
-        val binding: ViewFeedSearchRowBinding = ViewFeedSearchRowBinding.bind(itemView)
-    }
-
-    internal class ResultDiffCallback(private val oldList: List<FeedResult>,
-                                      private val newList: List<FeedResult>) : DiffUtil.Callback() {
-
-        override fun areItemsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
-            val oldFeedResult = oldList[oldItemPosition]
-            val newFeedResult = newList[newItemPosition]
-            return oldFeedResult.label == newFeedResult.label &&
-                    oldFeedResult.numberOfSubscriber == newFeedResult.numberOfSubscriber
-        }
-
-        override fun areContentsTheSame(oldItemPosition: Int, newItemPosition: Int): Boolean {
-            val oldFeedResult = oldList[oldItemPosition]
-            val newFeedResult = newList[newItemPosition]
-            return oldFeedResult.label == newFeedResult.label
-                    && oldFeedResult.tagline == newFeedResult.tagline
-        }
-
-        override fun getOldListSize(): Int = oldList.size
-
-        override fun getNewListSize(): Int = newList.size
-    }
-
-    interface OnFeedSearchResultClickListener {
-
-        fun onFeedSearchResultClickListener(result: FeedResult)
-    }
-}

clients/android/NewsBlur/src/com/newsblur/activity/InitActivity.kt

@@ -5,6 +5,7 @@ import android.os.Bundle
 import androidx.appcompat.app.AppCompatActivity
 import androidx.core.splashscreen.SplashScreen.Companion.installSplashScreen
 import androidx.lifecycle.lifecycleScope
+import com.newsblur.service.SubscriptionSyncService
 import com.newsblur.util.*
 
 /**
@@ -42,12 +43,12 @@ class InitActivity : AppCompatActivity() {
         upgradeCheck()
 
         // see if a user is already logged in; if so, jump to the Main activity
-        preferenceCheck()
+        userAuthCheck()
     }
 
-    private fun preferenceCheck() {
-        val preferences = getSharedPreferences(PrefConstants.PREFERENCES, MODE_PRIVATE)
-        if (preferences.getString(PrefConstants.PREF_COOKIE, null) != null) {
+    private fun userAuthCheck() {
+        if (PrefsUtils.hasCookie(this)) {
+            SubscriptionSyncService.schedule(this)
             val mainIntent = Intent(this, Main::class.java)
             startActivity(mainIntent)
         } else {

clients/android/NewsBlur/src/com/newsblur/activity/LoginProgress.kt

@@ -11,6 +11,7 @@ import androidx.lifecycle.lifecycleScope
 import com.newsblur.R
 import com.newsblur.databinding.ActivityLoginProgressBinding
 import com.newsblur.network.APIManager
+import com.newsblur.service.SubscriptionSyncService
 import com.newsblur.util.PrefsUtils
 import com.newsblur.util.UIUtils
 import com.newsblur.util.executeAsyncTask
@@ -60,6 +61,9 @@ class LoginProgress : FragmentActivity() {
                         val b = AnimationUtils.loadAnimation(this, R.anim.text_up)
                         binding.loginRetrievingFeeds.setText(R.string.login_retrieving_feeds)
                         binding.loginFeedProgress.startAnimation(b)
+
+                        SubscriptionSyncService.schedule(this)
+
                         val startMain = Intent(this, Main::class.java)
                         startActivity(startMain)
                     } else {

clients/android/NewsBlur/src/com/newsblur/activity/Main.java

@@ -340,7 +340,7 @@ public class Main extends NbActivity implements StateChangedListener, SwipeRefre
     }
 
     private void onClickAddButton() {
-        Intent i = new Intent(this, SearchForFeeds.class);
+        Intent i = new Intent(this, FeedSearchActivity.class);
         startActivity(i);
     }
 

clients/android/NewsBlur/src/com/newsblur/activity/Premium.kt

@@ -4,7 +4,6 @@ import android.graphics.Color
 import android.graphics.Paint
 import android.net.Uri
 import android.os.Bundle
-import android.text.TextUtils
 import android.text.util.Linkify
 import android.view.View
 import android.widget.TextView
@@ -12,82 +11,37 @@ import androidx.lifecycle.lifecycleScope
 import com.android.billingclient.api.*
 import com.newsblur.R
 import com.newsblur.databinding.ActivityPremiumBinding
-import com.newsblur.network.APIManager
-import com.newsblur.service.NBSyncService
+import com.newsblur.subscription.SubscriptionManager
+import com.newsblur.subscription.SubscriptionManagerImpl
+import com.newsblur.subscription.SubscriptionsListener
 import com.newsblur.util.*
 import nl.dionsegijn.konfetti.emitters.StreamEmitter
 import nl.dionsegijn.konfetti.models.Shape.Circle
 import nl.dionsegijn.konfetti.models.Shape.Square
 import nl.dionsegijn.konfetti.models.Size
-import java.text.DateFormat
-import java.text.SimpleDateFormat
 import java.util.*
 
 class Premium : NbActivity() {
 
     private lateinit var binding: ActivityPremiumBinding
-    private lateinit var billingClient: BillingClient
+    private lateinit var subscriptionManager: SubscriptionManager
 
-    private var subscriptionDetails: SkuDetails? = null
-    private var purchasedSubscription: Purchase? = null
+    private val subscriptionManagerListener = object : SubscriptionsListener {
 
-    private val acknowledgePurchaseResponseListener = AcknowledgePurchaseResponseListener { billingResult: BillingResult ->
-        when (billingResult.responseCode) {
-            BillingClient.BillingResponseCode.OK -> {
-                Log.d(this@Premium.localClassName, "acknowledgePurchaseResponseListener OK")
-                verifyUserSubscriptionStatus()
-            }
-            BillingClient.BillingResponseCode.BILLING_UNAVAILABLE -> {
-                // Billing API version is not supported for the type requested.
-                Log.d(this@Premium.localClassName, "acknowledgePurchaseResponseListener BILLING_UNAVAILABLE")
-            }
-            BillingClient.BillingResponseCode.SERVICE_UNAVAILABLE -> {
-                // Network connection is down.
-                Log.d(this@Premium.localClassName, "acknowledgePurchaseResponseListener SERVICE_UNAVAILABLE")
-            }
-            else -> {
-                // Handle any other error codes.
-                Log.d(this@Premium.localClassName, "acknowledgePurchaseResponseListener ERROR - message: " + billingResult.debugMessage)
-            }
-        }
-    }
-    private val purchaseUpdateListener = PurchasesUpdatedListener { billingResult: BillingResult, purchases: List<Purchase>? ->
-        if (billingResult.responseCode == BillingClient.BillingResponseCode.OK && purchases != null) {
-            Log.d(this@Premium.localClassName, "purchaseUpdateListener OK")
-            for (purchase in purchases) {
-                handlePurchase(purchase)
-            }
-        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.USER_CANCELED) {
-            // Handle an error caused by a user cancelling the purchase flow.
-            Log.d(this@Premium.localClassName, "purchaseUpdateListener USER_CANCELLED")
-        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.BILLING_UNAVAILABLE) {
-            // Billing API version is not supported for the type requested.
-            Log.d(this@Premium.localClassName, "purchaseUpdateListener BILLING_UNAVAILABLE")
-        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.SERVICE_UNAVAILABLE) {
-            // Network connection is down.
-            Log.d(this@Premium.localClassName, "purchaseUpdateListener SERVICE_UNAVAILABLE")
-        } else {
-            // Handle any other error codes.
-            Log.d(this@Premium.localClassName, "purchaseUpdateListener ERROR - message: " + billingResult.debugMessage)
-        }
-    }
-    private val billingClientStateListener: BillingClientStateListener = object : BillingClientStateListener {
-        override fun onBillingSetupFinished(billingResult: BillingResult) {
-            if (billingResult.responseCode == BillingClient.BillingResponseCode.OK) {
-                // The BillingClient is ready. You can query purchases here.
-                Log.d(this@Premium.localClassName, "onBillingSetupFinished OK")
-                retrievePlayStoreSubscriptions()
-                verifyUserSubscriptionStatus()
-            } else {
-                showSubscriptionDetailsError()
-            }
+        override fun onActiveSubscription(renewalMessage: String?) {
+            showActiveSubscriptionDetails(renewalMessage)
         }
 
-        override fun onBillingServiceDisconnected() {
-            Log.d(this@Premium.localClassName, "onBillingServiceDisconnected")
-            // Try to restart the connection on the next request to
-            // Google Play by calling the startConnection() method.
-            showSubscriptionDetailsError()
+        override fun onAvailableSubscription(skuDetails: SkuDetails) {
+            showAvailableSubscriptionDetails(skuDetails)
+        }
+
+        override fun onBillingConnectionReady() {
+            subscriptionManager.syncSubscriptionState()
+        }
+
+        override fun onBillingConnectionError(message: String?) {
+            showSubscriptionDetailsError(message)
         }
     }
 
@@ -96,7 +50,7 @@ class Premium : NbActivity() {
         binding = ActivityPremiumBinding.inflate(layoutInflater)
         setContentView(binding.root)
         setupUI()
-        setupBillingClient()
+        setupBilling()
     }
 
     private fun setupUI() {
@@ -113,129 +67,45 @@ class Premium : NbActivity() {
         FeedUtils.iconLoader!!.displayImage(AppConstants.SHILOH_PHOTO_URL, binding.imgShiloh)
     }
 
-    private fun setupBillingClient() {
-        billingClient = BillingClient.newBuilder(this)
-                .setListener(purchaseUpdateListener)
-                .enablePendingPurchases()
-                .build()
-        billingClient.startConnection(billingClientStateListener)
+    private fun setupBilling() {
+        subscriptionManager = SubscriptionManagerImpl(this, lifecycleScope)
+        subscriptionManager.startBillingConnection(subscriptionManagerListener)
     }
 
-    private fun verifyUserSubscriptionStatus() {
-        val hasNewsBlurSubscription = PrefsUtils.getIsPremium(this)
-        var playStoreSubscription: Purchase? = null
-        val result = billingClient.queryPurchases(BillingClient.SkuType.SUBS)
-        if (result.purchasesList != null) {
-            for (purchase in result.purchasesList!!) {
-                if (purchase.sku == AppConstants.PREMIUM_SKU) {
-                    playStoreSubscription = purchase
-                }
-            }
-        }
-        if (hasNewsBlurSubscription || playStoreSubscription != null) {
-            binding.containerGoingPremium.visibility = View.GONE
-            binding.containerGonePremium.visibility = View.VISIBLE
-            val expirationTimeMs = PrefsUtils.getPremiumExpire(this)
-            var renewalString: String? = null
-            if (expirationTimeMs == 0L) {
-                renewalString = getString(R.string.premium_subscription_no_expiration)
-            } else if (expirationTimeMs > 0) {
-                // date constructor expects ms
-                val expirationDate = Date(expirationTimeMs * 1000)
-                val dateFormat: DateFormat = SimpleDateFormat("EEE, MMMM d, yyyy", Locale.getDefault())
-                dateFormat.timeZone = TimeZone.getDefault()
-                renewalString = getString(R.string.premium_subscription_renewal, dateFormat.format(expirationDate))
-                if (playStoreSubscription != null && !playStoreSubscription.isAutoRenewing) {
-                    renewalString = getString(R.string.premium_subscription_expiration, dateFormat.format(expirationDate))
-                }
-            }
-            if (!TextUtils.isEmpty(renewalString)) {
-                binding.textSubscriptionRenewal.text = renewalString
-                binding.textSubscriptionRenewal.visibility = View.VISIBLE
-            }
-            showConfetti()
-        }
-        if (!hasNewsBlurSubscription && playStoreSubscription != null) {
-            purchasedSubscription = playStoreSubscription
-            notifyNewsBlurOfSubscription()
-        }
-    }
-
-    private fun retrievePlayStoreSubscriptions() {
-        val skuList: MutableList<String> = ArrayList(1)
-        // add sub SKUs from Play Store
-        skuList.add(AppConstants.PREMIUM_SKU)
-        val params = SkuDetailsParams.newBuilder()
-        params.setSkusList(skuList).setType(BillingClient.SkuType.SUBS)
-        billingClient.querySkuDetailsAsync(params.build()) { _: BillingResult?, skuDetailsList: List<SkuDetails>? ->
-            Log.d(this@Premium.localClassName, "SkuDetailsResponse")
-            processSkuDetailsList(skuDetailsList)
-        }
-    }
-
-    private fun processSkuDetailsList(skuDetailsList: List<SkuDetails>?) {
-        if (skuDetailsList != null) {
-            for (skuDetails in skuDetailsList) {
-                if (skuDetails.sku == AppConstants.PREMIUM_SKU) {
-                    Log.d(this@Premium.localClassName, "Sku detail: " + skuDetails.title + " | " + skuDetails.description + " | " + skuDetails.price + " | " + skuDetails.sku)
-                    subscriptionDetails = skuDetails
-                }
-            }
-        }
-        if (subscriptionDetails != null) {
-            showSubscriptionDetails()
-        } else {
-            showSubscriptionDetailsError()
-        }
-    }
-
-    private fun showSubscriptionDetailsError() {
-        binding.textLoading.setText(R.string.premium_subscription_details_error)
+    private fun showSubscriptionDetailsError(message: String?) {
+        binding.textLoading.text = message ?: getString(R.string.premium_subscription_details_error)
         binding.textLoading.visibility = View.VISIBLE
         binding.containerSub.visibility = View.GONE
     }
 
-    private fun showSubscriptionDetails() {
-        val price = (subscriptionDetails!!.priceAmountMicros / 1000f / 1000f).toDouble()
-        val currency = Currency.getInstance(subscriptionDetails!!.priceCurrencyCode)
+    private fun showAvailableSubscriptionDetails(skuDetails: SkuDetails) {
+        val price = (skuDetails.priceAmountMicros / 1000f / 1000f).toDouble()
+        val currency = Currency.getInstance(skuDetails.priceCurrencyCode)
         val currencySymbol = currency.getSymbol(Locale.getDefault())
         val pricingText = StringBuilder()
-        pricingText.append(subscriptionDetails!!.price)
+        pricingText.append(skuDetails.price)
         pricingText.append(" per year (")
         pricingText.append(currencySymbol)
         pricingText.append(String.format(Locale.getDefault(), "%.2f", price / 12))
         pricingText.append("/month)")
-        binding.textSubTitle.text = subscriptionDetails!!.title
+        binding.textSubTitle.text = skuDetails.title
         binding.textSubPrice.text = pricingText
         binding.textLoading.visibility = View.GONE
         binding.containerSub.visibility = View.VISIBLE
-        binding.containerSub.setOnClickListener { launchBillingFlow(subscriptionDetails!!) }
-    }
-
-    private fun launchBillingFlow(skuDetails: SkuDetails) {
-        Log.d(this@Premium.localClassName, "launchBillingFlow for sku: " + skuDetails.sku)
-        val billingFlowParams = BillingFlowParams.newBuilder()
-                .setSkuDetails(skuDetails)
-                .build()
-        billingClient.launchBillingFlow(this, billingFlowParams)
-    }
-
-    private fun handlePurchase(purchase: Purchase) {
-        Log.d(this@Premium.localClassName, "handlePurchase: " + purchase.orderId)
-        purchasedSubscription = purchase
-        if (purchase.purchaseState == Purchase.PurchaseState.PURCHASED && purchase.isAcknowledged) {
-            verifyUserSubscriptionStatus()
-        } else if (purchase.purchaseState == Purchase.PurchaseState.PURCHASED && !purchase.isAcknowledged) {
-            // need to acknowledge first time sub otherwise it will void
-            Log.d(this@Premium.localClassName, "acknowledge purchase: " + purchase.orderId)
-            val acknowledgePurchaseParams = AcknowledgePurchaseParams.newBuilder()
-                    .setPurchaseToken(purchase.purchaseToken)
-                    .build()
-            billingClient.acknowledgePurchase(acknowledgePurchaseParams, acknowledgePurchaseResponseListener)
+        binding.containerSub.setOnClickListener {
+            subscriptionManager.purchaseSubscription(this, skuDetails)
         }
     }
 
-    private fun showConfetti() {
+    private fun showActiveSubscriptionDetails(renewalMessage: String?) {
+        binding.containerGoingPremium.visibility = View.GONE
+        binding.containerGonePremium.visibility = View.VISIBLE
+
+        if (!renewalMessage.isNullOrEmpty()) {
+            binding.textSubscriptionRenewal.text = renewalMessage
+            binding.textSubscriptionRenewal.visibility = View.VISIBLE
+        }
+
         binding.konfetti.build()
                 .addColors(Color.YELLOW, Color.GREEN, Color.MAGENTA, Color.BLUE, Color.CYAN, Color.RED)
                 .setDirection(90.0)
@@ -246,22 +116,4 @@ class Premium : NbActivity() {
                 .setPosition(0f, binding.konfetti.width + 0f, -50f, -20f)
                 .streamFor(100, StreamEmitter.INDEFINITE)
     }
-
-    private fun notifyNewsBlurOfSubscription() {
-        if (purchasedSubscription != null) {
-            val apiManager = APIManager(this)
-            lifecycleScope.executeAsyncTask(
-                    doInBackground = {
-                        apiManager.saveReceipt(purchasedSubscription!!.orderId, purchasedSubscription!!.sku)
-                    },
-                    onPostExecute = {
-                        if (!it.isError) {
-                            NBSyncService.forceFeedsFolders()
-                            triggerSync()
-                        }
-                        finish()
-                    }
-            )
-        }
-    }
 }

clients/android/NewsBlur/src/com/newsblur/activity/Reading.kt

@@ -12,9 +12,9 @@ import android.widget.SeekBar
 import android.widget.SeekBar.OnSeekBarChangeListener
 import android.widget.Toast
 import androidx.fragment.app.FragmentManager
+import androidx.fragment.app.commit
+import androidx.lifecycle.ViewModelProvider
 import androidx.lifecycle.lifecycleScope
-import androidx.loader.app.LoaderManager
-import androidx.loader.content.Loader
 import androidx.viewpager.widget.ViewPager
 import androidx.viewpager.widget.ViewPager.OnPageChangeListener
 import com.google.android.material.progressindicator.CircularProgressIndicator
@@ -31,16 +31,17 @@ import com.newsblur.service.NBSyncService
 import com.newsblur.util.*
 import com.newsblur.util.PrefConstants.ThemeValue
 import com.newsblur.view.ReadingScrollView.ScrollChangeListener
+import com.newsblur.viewModel.StoriesViewModel
 import java.lang.Runnable
 import java.util.*
 import kotlin.math.abs
 
-abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeListener, ScrollChangeListener, LoaderManager.LoaderCallbacks<Cursor?>, ReadingFontChangedListener {
+abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeListener,
+        ScrollChangeListener, ReadingFontChangedListener {
 
     @JvmField
     var fs: FeedSet? = null
 
-    private val storiesMutex = Any()
     private var stories: Cursor? = null
 
     // Activities navigate to a particular story by hash.
@@ -58,11 +59,12 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
     private var overlayRangeBotPx = 0f
     private var lastVScrollPos = 0
 
-    private val pageHistory: MutableList<Story> = ArrayList()
+    private val pageHistory = mutableListOf<Story>()
 
     private lateinit var volumeKeyNavigation: VolumeKeyNavigation
     private lateinit var intelState: StateFilter
     private lateinit var binding: ActivityReadingBinding
+    private lateinit var storiesViewModel: StoriesViewModel
 
     override fun onNewIntent(intent: Intent) {
         super.onNewIntent(intent)
@@ -71,9 +73,9 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
 
     override fun onCreate(savedInstanceBundle: Bundle?) {
         super.onCreate(savedInstanceBundle)
+        storiesViewModel = ViewModelProvider(this).get(StoriesViewModel::class.java)
         binding = ActivityReadingBinding.inflate(layoutInflater)
         setContentView(binding.root)
-//        contentView = findViewById(android.R.id.content)
 
         try {
             fs = intent.getSerializableExtra(EXTRA_FEEDSET) as FeedSet?
@@ -106,38 +108,10 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
         intelState = PrefsUtils.getStateFilter(this)
         volumeKeyNavigation = PrefsUtils.getVolumeKeyNavigation(this)
 
-        // this value is expensive to compute but doesn't change during a single runtime
-        overlayRangeTopPx = UIUtils.dp2px(this, OVERLAY_RANGE_TOP_DP).toFloat()
-        overlayRangeBotPx = UIUtils.dp2px(this, OVERLAY_RANGE_BOT_DP).toFloat()
-
-        ViewUtils.setViewElevation(binding.readingOverlayLeft, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlayRight, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlayText, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlaySend, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlayProgress, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlayProgressLeft, OVERLAY_ELEVATION_DP)
-        ViewUtils.setViewElevation(binding.readingOverlayProgressRight, OVERLAY_ELEVATION_DP)
-
-        // this likes to default to 'on' for some platforms
-        enableProgressCircle(binding.readingOverlayProgressLeft, false)
-        enableProgressCircle(binding.readingOverlayProgressRight, false)
-
-        val fragmentManager = supportFragmentManager
-        var fragment = fragmentManager.findFragmentByTag(ReadingPagerFragment::class.java.name) as ReadingPagerFragment?
-        if (fragment == null) {
-            fragment = ReadingPagerFragment.newInstance()
-            val transaction = fragmentManager.beginTransaction()
-            transaction.add(R.id.activity_reading_container, fragment, ReadingPagerFragment::class.java.name)
-            transaction.commit()
-        }
-
-        binding.readingOverlayText.setOnClickListener { overlayTextClick() }
-        binding.readingOverlaySend.setOnClickListener { overlaySendClick() }
-        binding.readingOverlayLeft.setOnClickListener { overlayLeftClick() }
-        binding.readingOverlayRight.setOnClickListener { overlayRightClick() }
-        binding.readingOverlayProgress.setOnClickListener { overlayProgressCountClick() }
-
-        LoaderManager.getInstance(this).initLoader(0, null, this)
+        setupViews()
+        setupListeners()
+        setupObservers()
+        getActiveStoriesCursor(true)
     }
 
     override fun onSaveInstanceState(outState: Bundle) {
@@ -173,45 +147,78 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
         super.onPause()
     }
 
-    override fun onCreateLoader(loaderId: Int, bundle: Bundle?): Loader<Cursor?> {
-        if (fs == null) {
-            Log.e(this.javaClass.name, "can't create activity, no feedset ready")
-            // this is probably happening in a finalisation cycle or during a crash, pop the activity stack
-            finish()
+    private fun setupViews() {
+        // this value is expensive to compute but doesn't change during a single runtime
+        overlayRangeTopPx = UIUtils.dp2px(this, OVERLAY_RANGE_TOP_DP).toFloat()
+        overlayRangeBotPx = UIUtils.dp2px(this, OVERLAY_RANGE_BOT_DP).toFloat()
+
+        ViewUtils.setViewElevation(binding.readingOverlayLeft, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlayRight, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlayText, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlaySend, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlayProgress, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlayProgressLeft, OVERLAY_ELEVATION_DP)
+        ViewUtils.setViewElevation(binding.readingOverlayProgressRight, OVERLAY_ELEVATION_DP)
+
+        // this likes to default to 'on' for some platforms
+        enableProgressCircle(binding.readingOverlayProgressLeft, false)
+        enableProgressCircle(binding.readingOverlayProgressRight, false)
+
+        supportFragmentManager.commit {
+            val fragment =
+                    supportFragmentManager.findFragmentByTag(ReadingPagerFragment::class.java.name) as ReadingPagerFragment?
+                            ?: ReadingPagerFragment.newInstance()
+            add(R.id.activity_reading_container, fragment, ReadingPagerFragment::class.java.name)
         }
-        return FeedUtils.dbHelper!!.getActiveStoriesLoader(fs)
     }
 
-    override fun onLoaderReset(loader: Loader<Cursor?>) {}
+    private fun setupListeners() {
+        binding.readingOverlayText.setOnClickListener { overlayTextClick() }
+        binding.readingOverlaySend.setOnClickListener { overlaySendClick() }
+        binding.readingOverlayLeft.setOnClickListener { overlayLeftClick() }
+        binding.readingOverlayRight.setOnClickListener { overlayRightClick() }
+        binding.readingOverlayProgress.setOnClickListener { overlayProgressCountClick() }
+    }
 
-    override fun onLoadFinished(loader: Loader<Cursor?>, cursor: Cursor?) {
-        synchronized(storiesMutex) {
-            if (cursor == null) return
-            if (stopLoading) return
+    private fun setupObservers() {
+        storiesViewModel.activeStoriesLiveData.observe(this) {
+            setCursorData(it)
+        }
+    }
 
-            if (!FeedUtils.dbHelper!!.isFeedSetReady(fs)) {
-                com.newsblur.util.Log.i(this.javaClass.name, "stale load")
-                // the system can and will re-use activities, so during the initial mismatch of
-                // data, don't show the old stories
-                pager!!.visibility = View.INVISIBLE
-                binding.readingEmptyViewText.visibility = View.VISIBLE
-                stories = null
-                triggerRefresh(AppConstants.READING_STORY_PRELOAD)
-                return
+    private fun getActiveStoriesCursor(finishOnInvalidFs: Boolean = false) {
+        fs?.let {
+            storiesViewModel.getActiveStories(it)
+        } ?: run {
+            if (finishOnInvalidFs) {
+                Log.e(this.javaClass.name, "can't create activity, no feedset ready")
+                // this is probably happening in a finalisation cycle or during a crash, pop the activity stack
+                finish()
             }
+        }
+    }
 
-            if (readingAdapter != null) {
-                // swapCursor() will asynch process the new cursor and fully update the pager,
-                // update child fragments, and then call pagerUpdated()
-                readingAdapter!!.swapCursor(cursor, pager)
-            }
+    private fun setCursorData(cursor: Cursor) {
+        if (!FeedUtils.dbHelper!!.isFeedSetReady(fs)) {
+            com.newsblur.util.Log.i(this.javaClass.name, "stale load")
+            // the system can and will re-use activities, so during the initial mismatch of
+            // data, don't show the old stories
+            pager!!.visibility = View.INVISIBLE
+            binding.readingEmptyViewText.visibility = View.VISIBLE
+            stories = null
+            triggerRefresh(AppConstants.READING_STORY_PRELOAD)
+            return
+        }
 
-            stories = cursor
+        // swapCursor() will asynch process the new cursor and fully update the pager,
+        // update child fragments, and then call pagerUpdated()
+        readingAdapter?.swapCursor(cursor, pager)
 
-            com.newsblur.util.Log.d(this.javaClass.name, "loaded cursor with count: " + cursor.count)
-            if (cursor.count < 1) {
-                triggerRefresh(AppConstants.READING_STORY_PRELOAD)
-            }
+        stories = cursor
+
+        com.newsblur.util.Log.d(this.javaClass.name, "loaded cursor with count: " + cursor.count)
+        if (cursor.count < 1) {
+            triggerRefresh(AppConstants.READING_STORY_PRELOAD)
         }
     }
 
@@ -334,7 +341,6 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
         }
         if (updateType and UPDATE_STATUS != 0) {
             enableMainProgress(NBSyncService.isFeedSetSyncing(fs, this))
-//            if (binding!!.readingSyncStatus != null) {
             var syncStatus = NBSyncService.getSyncStatusMessage(this, true)
             if (syncStatus != null) {
                 if (AppConstants.VERBOSE_LOG) {
@@ -345,27 +351,15 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
             } else {
                 binding.readingSyncStatus.visibility = View.GONE
             }
-//            }
         }
         if (updateType and UPDATE_STORY != 0) {
-            updateCursor()
+            getActiveStoriesCursor()
             updateOverlayNav()
         }
 
         readingFragment?.handleUpdate(updateType)
     }
 
-    private fun updateCursor() {
-        synchronized(storiesMutex) {
-            try {
-                LoaderManager.getInstance(this).restartLoader(0, null, this)
-            } catch (ise: IllegalStateException) {
-                // our heavy use of async can race loader calls, which it will gripe about, but this
-                //  is only a refresh call, so dropping a refresh during creation is perfectly fine.
-            }
-        }
-    }
-
     // interface OnPageChangeListener
     override fun onPageScrollStateChanged(arg0: Int) {}
 
@@ -482,7 +476,6 @@ abstract class Reading : NbActivity(), OnPageChangeListener, OnSeekBarChangeList
     }
 
     private fun updateOverlayText() {
-//        if (binding!!.readingOverlayText == null) return
         runOnUiThread(Runnable {
             val item = readingFragment ?: return@Runnable
             if (item.selectedViewMode == DefaultFeedView.STORY) {

clients/android/NewsBlur/src/com/newsblur/database/BlurDatabaseHelper.java

@@ -1052,22 +1052,6 @@ public class BlurDatabaseHelper {
         synchronized (RW_MUTEX) {dbRW.insertOrThrow(DatabaseConstants.STORY_TEXT_TABLE, null, values);}
     }
 
-    /**
-     * Get a loader that always returns a null cursor, for fragments that know they will never
-     * have a result (such as muted feeds).
-     */
-    public Loader<Cursor> getNullLoader() {
-        return new AsyncTaskLoader<Cursor>(context) {
-            public Cursor loadInBackground() {return null;}
-        };
-    }
-        
-    public Loader<Cursor> getSocialFeedsLoader() {
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {return getSocialFeedsCursor(cancellationSignal);}
-        };
-    }
-
     public Cursor getSocialFeedsCursor(CancellationSignal cancellationSignal) {
         return query(false, DatabaseConstants.SOCIALFEED_TABLE, null, null, null, null, null, "UPPER(" + DatabaseConstants.SOCIAL_FEED_TITLE + ") ASC", null, cancellationSignal);
     }
@@ -1103,44 +1087,19 @@ public class BlurDatabaseHelper {
         return folders;
     }
 
-    public Loader<Cursor> getFoldersLoader() {
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {return getFoldersCursor(cancellationSignal);}
-        };
-    }
-
     public Cursor getFoldersCursor(CancellationSignal cancellationSignal) {
         return query(false, DatabaseConstants.FOLDER_TABLE, null, null, null, null, null, null, null, cancellationSignal);
     }
 
-    public Loader<Cursor> getFeedsLoader() {
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {return getFeedsCursor(cancellationSignal);}
-        };
-    }
-
     public Cursor getFeedsCursor(CancellationSignal cancellationSignal) {
         return query(false, DatabaseConstants.FEED_TABLE, null, null, null, null, null, "UPPER(" + DatabaseConstants.FEED_TITLE + ") ASC", null, cancellationSignal);
     }
 
-    public Loader<Cursor> getSavedStoryCountsLoader() {
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {return getSavedStoryCountsCursor(cancellationSignal);}
-        };
+    public Cursor getSavedStoryCountsCursor(CancellationSignal cancellationSignal) {
+        return query(false, DatabaseConstants.STARREDCOUNTS_TABLE, null, null, null, null, null, null, null, cancellationSignal);
     }
 
-    public Loader<Cursor> getSavedSearchLoader() {
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {return getSavedSearchCursor(cancellationSignal);}
-        };
-    }
-
-    private Cursor getSavedStoryCountsCursor(CancellationSignal cancellationSignal) {
-        Cursor c = query(false, DatabaseConstants.STARREDCOUNTS_TABLE, null, null, null, null, null, null, null, cancellationSignal);
-        return c;
-    }
-
-    private Cursor getSavedSearchCursor(CancellationSignal cancellationSignal) {
+    public Cursor getSavedSearchCursor(CancellationSignal cancellationSignal) {
         return query(false, DatabaseConstants.SAVED_SEARCH_TABLE, null, null, null, null,  null, null, null, cancellationSignal);
     }
 
@@ -1168,24 +1127,6 @@ public class BlurDatabaseHelper {
         return feedIds;
     }
 
-    public Loader<Cursor> getActiveStoriesLoader(final FeedSet fs) {
-        final StoryOrder order = PrefsUtils.getStoryOrder(context, fs);
-        return new QueryCursorLoader(context) {
-            protected Cursor createCursor() {
-                return getActiveStoriesCursor(fs, order, cancellationSignal);
-            }
-        };
-    }
-
-    public Loader<Cursor> getStoriesLoader(@Nullable final FeedSet fs) {
-        return new QueryCursorLoader(context) {
-            @Override
-            protected Cursor createCursor() {
-                return getStoriesCursor(fs, cancellationSignal);
-            }
-        };
-    }
-
     private Cursor getStoriesCursor(@Nullable FeedSet fs, CancellationSignal cancellationSignal) {
         StringBuilder q = new StringBuilder(DatabaseConstants.STORY_QUERY_BASE_0);
 
@@ -1204,7 +1145,8 @@ public class BlurDatabaseHelper {
         return rawQuery(q.toString(), null, cancellationSignal);
     }
 
-    private Cursor getActiveStoriesCursor(FeedSet fs, StoryOrder order, CancellationSignal cancellationSignal) {
+    public Cursor getActiveStoriesCursor(FeedSet fs, CancellationSignal cancellationSignal) {
+        final StoryOrder order = PrefsUtils.getStoryOrder(context, fs);
         // get the stories for this FS
         Cursor result = getActiveStoriesCursorNoPrep(fs, order, cancellationSignal);
         // if the result is blank, try to prime the session table with existing stories, in case we

clients/android/NewsBlur/src/com/newsblur/database/QueryCursorLoader.java

@@ -1,133 +0,0 @@
-package com.newsblur.database;
-
-import android.content.Context;
-import android.database.Cursor;
-import android.os.CancellationSignal;
-import android.os.OperationCanceledException;
-import androidx.loader.content.AsyncTaskLoader;
-
-import com.newsblur.util.AppConstants;
-
-/**
- * A partial copy of android.content.CursorLoader with the bits related to ContentProviders
- * gutted out so plain old SQLiteDatabase queries can be used where a ContentProvider is
- * contraindicated. (Why this isn't in core Android I will never understand) Also fixes
- * several bugs with how LoaderManagers interact with AsyncTaskLoaders on several platforms.
- */
-public abstract class QueryCursorLoader extends AsyncTaskLoader<Cursor> {
-
-    // we hold onto a copy of any cursor vended so we can auto-close it, per the contract of a Loader
-    private Cursor cursor;
-    // we create and manage a cancellation hook since SQLite support it and it lets us quickly catch up when behind
-    protected CancellationSignal cancellationSignal;
-
-    public QueryCursorLoader(Context context) {
-        super(context);
-    }
-
-    /**
-     * Subclasses (generally anonymous) must actually provide the code to load the cursor, we just
-     * handly lifecyle management.
-     */
-    protected abstract Cursor createCursor();
-
-    // this is the method that AsyncTaskLoader actually calls to the the data object
-    @Override
-    public Cursor loadInBackground() {
-        synchronized (this) {
-            if (isLoadInBackgroundCanceled()) {
-                throw new OperationCanceledException();
-            }
-            cancellationSignal = new CancellationSignal();
-        }
-        try {
-            long startTime = System.nanoTime();
-            int count = -1;
-            Cursor c = createCursor();
-            if (c != null) {
-                // this call to getCount is *not* just for the instrumentation, it ensures the cursor is fully ready before
-                // being called back.  if the instrumentation is ever removed, do not remove this call.
-                count = c.getCount();
-            }
-            if (AppConstants.VERBOSE_LOG) {
-                long time = System.nanoTime() - startTime;
-                com.newsblur.util.Log.d(this.getClass().getName(), "cursor load: " + (time/1000000L) + "ms to load " + count + " rows");
-            }
-            return c;
-        } finally {
-            synchronized (this) {
-                cancellationSignal = null;
-            }
-        }
-    }
-
-    // this is a hook to try and actively cancel an in-flight load. cancellation flagging is handled elsewhere
-    @Override
-    public void cancelLoadInBackground() {
-        super.cancelLoadInBackground();
-        synchronized (this) {
-            if (cancellationSignal != null) {
-                cancellationSignal.cancel();
-                cancellationSignal = null;
-            }
-        }
-    }
-
-    // a hook for when data are delivered that lets us snag a copy so we can manage it
-    @Override
-    public void deliverResult(Cursor cursor) {
-        if (isReset()) {
-            clearCursor();
-            return;
-        }
-        Cursor oldCursor = this.cursor;
-        this.cursor = cursor;
-        if (isStarted()) {
-            super.deliverResult(cursor);
-        }
-        if (oldCursor != null && oldCursor != this.cursor && !oldCursor.isClosed()) {
-            oldCursor.close();
-        }
-    } 
-
-    @Override
-    protected void onStartLoading() {
-        if (cursor != null) {
-            // if we already have a cursor and haven't been reset, use it!
-            deliverResult(cursor);
-        }
-        // if we had nothing or have a pending change, reload
-        if ((cursor == null) || takeContentChanged()) {
-            forceLoad();
-        }
-    }
-
-    @Override
-    protected void onStopLoading() {
-        // not that we do *not* clear data in this hook. the framework may tell us to stop loading
-        // but still request our data later.  this isn't a reset.
-        cancelLoad();
-    }
-
-    @Override
-    public void onCanceled(Cursor cursor) {
-        clearCursor();
-    }
-
-    @Override
-    protected void onReset() {
-        super.onReset();
-        onStopLoading();
-        // note that the stock CursorLoader here closes the cursor early rather than waiting for the
-        // rest of the reset->stop->cancel->cancelled cycle, which can cause contexts to briefly have
-        // a closed cursor but no replacement.
-    }
-
-    private void clearCursor() {
-        if (cursor != null && !cursor.isClosed()) {
-            cursor.close();
-        }
-        cursor = null;
-    }
-
-}

clients/android/NewsBlur/src/com/newsblur/domain/FeedResult.java

@@ -1,22 +0,0 @@
-package com.newsblur.domain;
-
-import com.google.gson.annotations.SerializedName;
-
-public class FeedResult {
-
-	@SerializedName("num_subscribers")
-	public int numberOfSubscriber;
-	
-	@SerializedName("favicon_color")
-	public String faviconColor;
-	
-	@SerializedName("value")
-	public String url;
-	
-	public String tagline;
-	
-	public String label;
-	
-	public String favicon;
-	
-}

clients/android/NewsBlur/src/com/newsblur/domain/FeedResult.kt

@@ -0,0 +1,21 @@
+package com.newsblur.domain
+
+import com.google.gson.annotations.SerializedName
+import com.newsblur.network.APIConstants
+
+data class FeedResult(
+        @SerializedName("id")
+        val id: Int = 0,
+        @SerializedName("tagline")
+        val tagline: String? = null,
+        @SerializedName("label")
+        val label: String,
+        @SerializedName("num_subscribers")
+        val numberOfSubscriber: Int = 0,
+        @SerializedName("value")
+        val url: String,
+) {
+
+    val faviconUrl: String
+        get() = "${APIConstants.buildUrl(APIConstants.PATH_FEED_FAVICON_URL)}$id"
+}

clients/android/NewsBlur/src/com/newsblur/fragment/FolderListFragment.java

@@ -6,17 +6,15 @@ import java.util.Set;
 
 import android.content.Intent;
 import android.content.SharedPreferences;
-import android.database.Cursor;
 import android.os.Bundle;
 
 import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
 import androidx.core.content.ContextCompat;
 import androidx.fragment.app.DialogFragment;
-import androidx.loader.app.LoaderManager;
-import androidx.loader.content.Loader;
+import androidx.lifecycle.ViewModelProvider;
 
 import android.os.Handler;
-import android.util.Log;
 import android.view.ContextMenu;
 import android.view.ContextMenu.ContextMenuInfo;
 import android.view.LayoutInflater;
@@ -58,20 +56,15 @@ import com.newsblur.util.PrefConstants;
 import com.newsblur.util.PrefsUtils;
 import com.newsblur.util.StateFilter;
 import com.newsblur.util.UIUtils;
+import com.newsblur.viewModel.AllFoldersViewModel;
 
 public class FolderListFragment extends NbFragment implements OnCreateContextMenuListener, 
-                                                              LoaderManager.LoaderCallbacks<Cursor>,
-                                                              OnChildClickListener, 
+                                                              OnChildClickListener,
                                                               OnGroupClickListener,
                                                               OnGroupCollapseListener,
                                                               OnGroupExpandListener {
 
-    private static final int SOCIALFEEDS_LOADER = 1;
-    private static final int FOLDERS_LOADER = 2;
-    private static final int FEEDS_LOADER = 3;
-    private static final int SAVEDCOUNT_LOADER = 4;
-    private static final int SAVED_SEARCH_LOADER = 5;
-
+    private AllFoldersViewModel allFoldersViewModel;
 	private FolderListAdapter adapter;
 	public StateFilter currentState = StateFilter.SOME;
 	private SharedPreferences sharedPreferences;
@@ -85,6 +78,7 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
 	@Override
 	public void onCreate(Bundle savedInstanceState) {
 		super.onCreate(savedInstanceState);
+		allFoldersViewModel = new ViewModelProvider(this).get(AllFoldersViewModel.class);
         currentState = PrefsUtils.getStateFilter(getActivity());
 		adapter = new FolderListAdapter(getActivity(), currentState);
         sharedPreferences = getActivity().getSharedPreferences(PrefConstants.PREFERENCES, 0);
@@ -93,6 +87,12 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
         // ping from the sync service indicating that it has initialised
     }
 
+    @Override
+    public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) {
+        super.onViewCreated(view, savedInstanceState);
+        setupObservers();
+    }
+
     @Override
     public void onResume() {
         super.onResume();
@@ -103,93 +103,40 @@ public class FolderListFragment extends NbFragment implements OnCreateContextMen
         }
     }
 
-	@NonNull
-    @Override
-	public Loader<Cursor> onCreateLoader(int id, Bundle args) {
-        switch (id) {
-            case SOCIALFEEDS_LOADER:
-                return FeedUtils.dbHelper.getSocialFeedsLoader();
-            case FOLDERS_LOADER:
-                return FeedUtils.dbHelper.getFoldersLoader();
-            case FEEDS_LOADER:
-                return FeedUtils.dbHelper.getFeedsLoader();
-            case SAVEDCOUNT_LOADER:
-                return FeedUtils.dbHelper.getSavedStoryCountsLoader();
-            case SAVED_SEARCH_LOADER:
-                return FeedUtils.dbHelper.getSavedSearchLoader();
-            default:
-                throw new IllegalArgumentException("unknown loader created");
-        }
-	}
-
-    @Override
-	public void onLoadFinished(@NonNull Loader<Cursor> loader, Cursor cursor) {
-        if (cursor == null) return;
-        try {
-            switch (loader.getId()) {
-                case SOCIALFEEDS_LOADER:
-                    adapter.setSocialFeedCursor(cursor);
-                    pushUnreadCounts();
-                    break;
-                case FOLDERS_LOADER:
-                    adapter.setFoldersCursor(cursor);
-                    pushUnreadCounts();
-                    checkOpenFolderPreferences();
-                    break;
-                case FEEDS_LOADER:
-                    adapter.setFeedCursor(cursor);
-                    checkOpenFolderPreferences();
-                    firstCursorSeenYet = true;
-                    pushUnreadCounts();
-                    checkAccountFeedsLimit();
-                    break;
-                case SAVEDCOUNT_LOADER:
-                    adapter.setStarredCountCursor(cursor);
-                    break;
-                case SAVED_SEARCH_LOADER:
-                    adapter.setSavedSearchesCursor(cursor);
-                    break;
-                default:
-                    throw new IllegalArgumentException("unknown loader created");
-            }
-        } catch (Exception e) {
-            // for complex folder sets, these ops can take so long that they butt heads
-            // with the destruction of the fragment and adapter. crashes can ensue.
-            Log.w(this.getClass().getName(), "failed up update fragment state", e);
-        }
+    private void setupObservers() {
+        allFoldersViewModel.getSocialFeeds().observe(getViewLifecycleOwner(), cursor -> {
+            adapter.setSocialFeedCursor(cursor);
+            pushUnreadCounts();
+        });
+        allFoldersViewModel.getFolders().observe(getViewLifecycleOwner(), cursor -> {
+            adapter.setFoldersCursor(cursor);
+            pushUnreadCounts();
+        });
+        allFoldersViewModel.getFeeds().observe(getViewLifecycleOwner(), cursor -> {
+            adapter.setFeedCursor(cursor);
+            checkOpenFolderPreferences();
+            firstCursorSeenYet = true;
+            pushUnreadCounts();
+            checkAccountFeedsLimit();
+        });
+        allFoldersViewModel.getSavedStoryCounts().observe(getViewLifecycleOwner(), cursor ->
+                adapter.setStarredCountCursor(cursor));
+        allFoldersViewModel.getSavedSearch().observe(getViewLifecycleOwner(), cursor ->
+                adapter.setSavedSearchesCursor(cursor));
     }
 
-	@Override
-	public void onLoaderReset(@NonNull Loader<Cursor> loader) {
-		; // our adapter doesn't hold on to cursors
-	}
-
 	public void hasUpdated() {
         if (isAdded()) {
+            allFoldersViewModel.getData();
             com.newsblur.util.Log.d(this, "loading feeds in mode: " + currentState);
-            try {
-                LoaderManager.getInstance(this).restartLoader(SOCIALFEEDS_LOADER, null, this);
-                LoaderManager.getInstance(this).restartLoader(FOLDERS_LOADER, null, this);
-                LoaderManager.getInstance(this).restartLoader(FEEDS_LOADER, null, this);
-                LoaderManager.getInstance(this).restartLoader(SAVEDCOUNT_LOADER, null, this);
-                LoaderManager.getInstance(this).restartLoader(SAVED_SEARCH_LOADER, null, this);
-            } catch (Exception e) {
-                // on heavily loaded devices, the time between isAdded() going false
-                // and the loader subsystem shutting down can be nontrivial, causing
-                // IllegalStateExceptions to be thrown here.
-            }
         }
 	}
 
     public synchronized void startLoaders() {
         if (isAdded()) {
-            if (LoaderManager.getInstance(this).getLoader(FOLDERS_LOADER) == null) {
-                // if the loaders haven't yet been created, do so
-                LoaderManager.getInstance(this).initLoader(SOCIALFEEDS_LOADER, null, this);
-                LoaderManager.getInstance(this).initLoader(FOLDERS_LOADER, null, this);
-                LoaderManager.getInstance(this).initLoader(FEEDS_LOADER, null, this);
-                LoaderManager.getInstance(this).initLoader(SAVEDCOUNT_LOADER, null, this);
-                LoaderManager.getInstance(this).initLoader(SAVED_SEARCH_LOADER, null, this);
+            if (allFoldersViewModel.getFolders().getValue() == null) {
+                // if the data haven't yet been fetched, do so
+                allFoldersViewModel.getData();
             }
         }
     }

clients/android/NewsBlur/src/com/newsblur/fragment/ItemSetFragment.java

@@ -6,10 +6,10 @@ import android.graphics.Rect;
 import android.os.Bundle;
 import android.os.Parcelable;
 
+import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.core.content.ContextCompat;
-import androidx.loader.app.LoaderManager;
-import androidx.loader.content.Loader;
+import androidx.lifecycle.ViewModelProvider;
 import androidx.recyclerview.widget.GridLayoutManager;
 import androidx.recyclerview.widget.RecyclerView;
 import android.view.GestureDetector;
@@ -37,10 +37,10 @@ import com.newsblur.util.ThumbnailStyle;
 import com.newsblur.util.UIUtils;
 import com.newsblur.util.ViewUtils;
 import com.newsblur.view.ProgressThrobber;
+import com.newsblur.viewModel.StoriesViewModel;
 
-public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderCallbacks<Cursor> {
+public class ItemSetFragment extends NbFragment {
 
-	public static int ITEMLIST_LOADER = 0x01;
     private static final String BUNDLE_GRIDSTATE = "gridstate";
 
     protected boolean cursorSeenYet = false; // have we yet seen a valid cursor for our particular feedset?
@@ -74,6 +74,7 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
 
     private FragmentItemgridBinding binding;
     private RowFleuronBinding fleuronBinding;
+    private StoriesViewModel storiesViewModel;
 
 	public static ItemSetFragment newInstance() {
 		ItemSetFragment fragment = new ItemSetFragment();
@@ -85,7 +86,7 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
     @Override
     public void onCreate(@Nullable Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
-        LoaderManager.getInstance(this).initLoader(ITEMLIST_LOADER, null, this);
+        storiesViewModel = new ViewModelProvider(this).get(StoriesViewModel.class);
     }
 
     @Override
@@ -147,7 +148,7 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
                 updateStyle();
             }
         });
-    
+
         StoryListStyle listStyle = PrefsUtils.getStoryListStyle(getActivity(), getFeedSet());
 
         calcColumnCount(listStyle);
@@ -193,6 +194,22 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
 		return v;
 	}
 
+    @Override
+    public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) {
+        super.onViewCreated(view, savedInstanceState);
+        storiesViewModel.getActiveStoriesLiveData().observe(getViewLifecycleOwner(), this::setCursor);
+
+        FeedSet fs = getFeedSet();
+        if (fs == null) {
+            com.newsblur.util.Log.e(this.getClass().getName(), "can't create fragment, no feedset ready");
+            // this is probably happening in a finalisation cycle or during a crash, pop the activity stack
+            try {
+                getActivity().finish();
+            } catch (Exception ignored) {
+            }
+        }
+    }
+
     protected void triggerRefresh(int desiredStoryCount, Integer totalSeen) {
         // ask the sync service for as many stories as we want
         boolean gotSome = NBSyncService.requestMoreForFeed(getFeedSet(), desiredStoryCount, totalSeen);
@@ -228,32 +245,29 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
     }
 
 	public void hasUpdated() {
-        if (isAdded()) {
-		    LoaderManager.getInstance(this).restartLoader(ITEMLIST_LOADER , null, this);
+        FeedSet fs = getFeedSet();
+        if (isAdded() && fs != null) {
+            storiesViewModel.getActiveStories(fs);
         }
 	}
 
-	@Override
-	public Loader<Cursor> onCreateLoader(int arg0, Bundle arg1) {
-        FeedSet fs = getFeedSet();
-        if (fs == null) {
-            com.newsblur.util.Log.e(this.getClass().getName(), "can't create fragment, no feedset ready");
-            // this is probably happening in a finalisation cycle or during a crash, pop the activity stack
-            try {
-                getActivity().finish();
-            } catch (Exception e) {
-
-            }
-            return FeedUtils.dbHelper.getNullLoader();
+    protected void updateAdapter(@Nullable Cursor cursor) {
+        adapter.swapCursor(cursor, binding.itemgridfragmentGrid, gridState);
+        gridState = null;
+        adapter.updateFeedSet(getFeedSet());
+        if ((cursor != null) && (cursor.getCount() > 0)) {
+            binding.emptyView.setVisibility(View.INVISIBLE);
         } else {
-            return FeedUtils.dbHelper.getActiveStoriesLoader(getFeedSet());
+            binding.emptyView.setVisibility(View.VISIBLE);
         }
+
+        // though we have stories, we might not yet have as many as we want
+        ensureSufficientStories();
     }
 
-    @Override
-	public synchronized void onLoadFinished(Loader<Cursor> loader, Cursor cursor) {
-		if (cursor != null) {
-            if (! FeedUtils.dbHelper.isFeedSetReady(getFeedSet())) {
+    private void setCursor(Cursor cursor) {
+        if (cursor != null) {
+            if (!FeedUtils.dbHelper.isFeedSetReady(getFeedSet())) {
                 // the DB hasn't caught up yet from the last story list; don't display stale stories.
                 com.newsblur.util.Log.i(this.getClass().getName(), "stale load");
                 updateAdapter(null);
@@ -268,26 +282,8 @@ public class ItemSetFragment extends NbFragment implements LoaderManager.LoaderC
             }
 		}
         updateLoadingIndicators();
-	}
-
-    protected void updateAdapter(Cursor cursor) {
-        adapter.swapCursor(cursor, binding.itemgridfragmentGrid, gridState);
-        gridState = null;
-        adapter.updateFeedSet(getFeedSet());
-        if ((cursor != null) && (cursor.getCount() > 0)) {
-            binding.emptyView.setVisibility(View.INVISIBLE);
-        } else {
-            binding.emptyView.setVisibility(View.VISIBLE);
-        }
-
-        // though we have stories, we might not yet have as many as we want
-        ensureSufficientStories();
     }
 
-	@Override
-	public void onLoaderReset(Loader<Cursor> loader) {
-	}
-
     private void updateLoadingIndicators() {
         calcFleuronPadding();
 

clients/android/NewsBlur/src/com/newsblur/fragment/ReadingItemFragment.kt

@@ -25,7 +25,6 @@ import com.newsblur.databinding.IncludeReadingItemCommentBinding
 import com.newsblur.domain.Classifier
 import com.newsblur.domain.Story
 import com.newsblur.domain.UserDetails
-import com.newsblur.fragment.StoryUserTagsFragment.Companion.newInstance
 import com.newsblur.network.APIManager
 import com.newsblur.service.NBSyncReceiver.Companion.UPDATE_INTEL
 import com.newsblur.service.NBSyncReceiver.Companion.UPDATE_SOCIAL
@@ -488,7 +487,7 @@ class ReadingItemFragment : NbFragment(), PopupMenu.OnMenuItemClickListener {
                     chip.chipIcon = ContextCompat.getDrawable(requireContext(), R.drawable.ic_add_gray75)
                 }
                 v.setOnClickListener {
-                    val userTagsFragment = newInstance(story!!, fs!!)
+                    val userTagsFragment = StoryUserTagsFragment.newInstance(story!!, fs!!)
                     userTagsFragment.show(childFragmentManager, StoryUserTagsFragment::class.java.name)
                 }
                 binding.readingItemUserTags.addView(v)

clients/android/NewsBlur/src/com/newsblur/fragment/StoryUserTagsFragment.kt

@@ -9,8 +9,7 @@ import android.widget.Toast
 import androidx.appcompat.app.AlertDialog
 import androidx.core.view.isVisible
 import androidx.fragment.app.DialogFragment
-import androidx.loader.app.LoaderManager
-import androidx.loader.content.Loader
+import androidx.lifecycle.ViewModelProvider
 import com.newsblur.R
 import com.newsblur.databinding.DialogStoryUserTagsBinding
 import com.newsblur.databinding.RowSavedTagBinding
@@ -20,16 +19,18 @@ import com.newsblur.service.NBSyncService
 import com.newsblur.util.FeedSet
 import com.newsblur.util.FeedUtils
 import com.newsblur.util.TagsAdapter
+import com.newsblur.viewModel.StoryUserTagsViewModel
 import java.util.*
 import kotlin.collections.ArrayList
 import kotlin.collections.HashMap
 import kotlin.collections.HashSet
 
-class StoryUserTagsFragment : DialogFragment(), LoaderManager.LoaderCallbacks<Cursor>, TagsAdapter.OnTagClickListener {
+class StoryUserTagsFragment : DialogFragment(), TagsAdapter.OnTagClickListener {
 
     private lateinit var story: Story
     private lateinit var fs: FeedSet
     private lateinit var binding: DialogStoryUserTagsBinding
+    private lateinit var storyUserTagsViewModel: StoryUserTagsViewModel
 
     private lateinit var otherTagsAdapter: TagsAdapter
     private lateinit var savedTagsAdapter: TagsAdapter
@@ -40,7 +41,6 @@ class StoryUserTagsFragment : DialogFragment(), LoaderManager.LoaderCallbacks<Cu
 
     companion object {
 
-        @JvmStatic
         fun newInstance(story: Story, fs: FeedSet): StoryUserTagsFragment {
             val fragment = StoryUserTagsFragment()
             val args = Bundle()
@@ -51,24 +51,11 @@ class StoryUserTagsFragment : DialogFragment(), LoaderManager.LoaderCallbacks<Cu
         }
     }
 
-    override fun onCreateLoader(id: Int, args: Bundle?): Loader<Cursor> =
-            FeedUtils.dbHelper!!.savedStoryCountsLoader
-
-    override fun onLoadFinished(loader: Loader<Cursor>, cursor: Cursor) {
-        if (!cursor.isBeforeFirst) return
-        val starredTags = ArrayList<StarredCount>()
-        while (cursor.moveToNext()) {
-            val sc = StarredCount.fromCursor(cursor)
-            if (sc.tag != null && !sc.isTotalCount) {
-                starredTags.add(sc)
-            }
-        }
-        Collections.sort(starredTags, StarredCount.StarredCountComparatorByTag)
-        setTags(starredTags)
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        storyUserTagsViewModel = ViewModelProvider(this).get(StoryUserTagsViewModel::class.java)
     }
 
-    override fun onLoaderReset(loader: Loader<Cursor>) {}
-
     override fun onCreateDialog(savedInstanceState: Bundle?): Dialog {
         super.onCreateDialog(savedInstanceState)
         val view = layoutInflater.inflate(R.layout.dialog_story_user_tags, null)
@@ -82,7 +69,9 @@ class StoryUserTagsFragment : DialogFragment(), LoaderManager.LoaderCallbacks<Cu
         story = requireArguments().getSerializable("story") as Story
         fs = requireArguments().getSerializable("feed_set") as FeedSet
 
-        LoaderManager.getInstance(this).initLoader(0, null, this)
+        storyUserTagsViewModel.savedStoryCountsLiveData.observe(this) {
+            setCursor(it)
+        }
 
         binding.textAddNewTag.setOnClickListener {
             if (binding.containerAddTag.isVisible) {
@@ -127,9 +116,24 @@ class StoryUserTagsFragment : DialogFragment(), LoaderManager.LoaderCallbacks<Cu
             binding.containerStoryTags.visibility = View.GONE
         }
 
+        storyUserTagsViewModel.getSavedStoryCounts()
+
         return builder.create()
     }
 
+    private fun setCursor(cursor: Cursor) {
+        if (!cursor.isBeforeFirst) return
+        val starredTags = ArrayList<StarredCount>()
+        while (cursor.moveToNext()) {
+            val sc = StarredCount.fromCursor(cursor)
+            if (sc.tag != null && !sc.isTotalCount) {
+                starredTags.add(sc)
+            }
+        }
+        Collections.sort(starredTags, StarredCount.StarredCountComparatorByTag)
+        setTags(starredTags)
+    }
+
     private fun processNewTag(newTag: StarredCount) {
         var foundExistingTag = false
         if (otherTags.contains(newTag.tag)) {

clients/android/NewsBlur/src/com/newsblur/network/APIConstants.java

@@ -81,6 +81,7 @@ public class APIConstants {
     public static final String PATH_RENAME_FOLDER = "/reader/rename_folder";
     public static final String PATH_SAVE_RECEIPT = "/profile/save_android_receipt";
     public static final String PATH_FEED_STATISTICS = "/rss_feeds/statistics_embedded/";
+    public static final String PATH_FEED_FAVICON_URL = "/rss_feeds/icon/";
 
     public static String buildUrl(String path) {
         return CurrentUrlBase + path;

clients/android/NewsBlur/src/com/newsblur/service/SubscriptionSyncService.kt

@@ -0,0 +1,81 @@
+package com.newsblur.service
+
+import android.app.job.JobInfo
+import android.app.job.JobParameters
+import android.app.job.JobScheduler
+import android.app.job.JobService
+import android.content.ComponentName
+import android.content.Context
+import com.newsblur.subscription.SubscriptionManagerImpl
+import com.newsblur.util.AppConstants
+import com.newsblur.util.Log
+import com.newsblur.util.NBScope
+import com.newsblur.util.PrefsUtils
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+/**
+ * Service to sync user subscription with NewsBlur backend.
+ *
+ * Mostly interested in handling the state where there is an active
+ * subscription in Play Store but NewsBlur doesn't know about it.
+ * This could occur when the user has renewed the subscription
+ * via Play Store.
+ */
+class SubscriptionSyncService : JobService() {
+
+    override fun onStartJob(params: JobParameters?): Boolean {
+        Log.d(this, "onStartJob")
+        if (!PrefsUtils.hasCookie(this)) {
+            // no user authenticated
+            return false
+        }
+
+        NBScope.launch(Dispatchers.Default) {
+            val subscriptionManager = SubscriptionManagerImpl(this@SubscriptionSyncService, this)
+            val job = subscriptionManager.syncActiveSubscription()
+            job.invokeOnCompletion {
+                Log.d(this, "sync active subscription completed.")
+                // manually trigger jobFinished after work is done
+                jobFinished(params, false)
+            }
+        }
+
+        return true // returning true due to background thread work
+    }
+
+    override fun onStopJob(params: JobParameters?): Boolean = false
+
+    companion object {
+
+        private const val JOB_ID = 2021
+
+        private fun createJobInfo(context: Context): JobInfo = JobInfo.Builder(JOB_ID,
+                ComponentName(context, SubscriptionSyncService::class.java))
+                .apply {
+                    // sync every 24 hours
+                    setPeriodic(AppConstants.BG_SUBSCRIPTION_SYNC_CYCLE_MILLIS)
+                    setRequiredNetworkType(JobInfo.NETWORK_TYPE_ANY)
+                    setBackoffCriteria(JobInfo.DEFAULT_INITIAL_BACKOFF_MILLIS, JobInfo.BACKOFF_POLICY_EXPONENTIAL)
+                    setPersisted(true)
+                }.build()
+
+        fun schedule(context: Context) {
+            val jobScheduler = context.getSystemService(Context.JOB_SCHEDULER_SERVICE) as JobScheduler
+            val job = jobScheduler.allPendingJobs.find { it.id == JOB_ID }
+            if (job == null) {
+                val result: Int = jobScheduler.schedule(createJobInfo(context))
+                Log.d(this, "Scheduled subscription result: ${if (result == JobScheduler.RESULT_FAILURE) "failed" else "completed"}")
+            }
+        }
+
+        @JvmStatic
+        fun cancel(context: Context) {
+            val jobScheduler = context.getSystemService(Context.JOB_SCHEDULER_SERVICE) as JobScheduler
+            jobScheduler.allPendingJobs.find { it.id == JOB_ID }?.let {
+                jobScheduler.cancel(JOB_ID)
+                Log.d(this, "Cancel sync job.")
+            }
+        }
+    }
+}

clients/android/NewsBlur/src/com/newsblur/subscription/SubscriptionManager.kt

@@ -0,0 +1,298 @@
+package com.newsblur.subscription
+
+import android.app.Activity
+import android.content.Context
+import com.android.billingclient.api.AcknowledgePurchaseParams
+import com.android.billingclient.api.AcknowledgePurchaseResponseListener
+import com.android.billingclient.api.BillingClient
+import com.android.billingclient.api.BillingClientStateListener
+import com.android.billingclient.api.BillingFlowParams
+import com.android.billingclient.api.BillingResult
+import com.android.billingclient.api.Purchase
+import com.android.billingclient.api.PurchasesUpdatedListener
+import com.android.billingclient.api.SkuDetails
+import com.android.billingclient.api.SkuDetailsParams
+import com.newsblur.R
+import com.newsblur.network.APIManager
+import com.newsblur.service.NBSyncService
+import com.newsblur.util.AppConstants
+import com.newsblur.util.FeedUtils
+import com.newsblur.util.Log
+import com.newsblur.util.NBScope
+import com.newsblur.util.PrefsUtils
+import com.newsblur.util.executeAsyncTask
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Deferred
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import java.text.DateFormat
+import java.text.SimpleDateFormat
+import java.util.*
+
+interface SubscriptionManager {
+
+    /**
+     * Open connection to Play Store to retrieve
+     * purchases and subscriptions.
+     */
+    fun startBillingConnection(listener: SubscriptionsListener? = null)
+
+    /**
+     * Generated subscription state by retrieve all available subscriptions
+     * and checking whether the user has an active subscription.
+     *
+     * Subscriptions are configured via the Play Store console.
+     */
+    fun syncSubscriptionState()
+
+    /**
+     * Launch the billing flow overlay for a specific subscription.
+     * @param activity Activity on which the billing overlay will be displayed.
+     * @param skuDetails Subscription details for the intended purchases.
+     */
+    fun purchaseSubscription(activity: Activity, skuDetails: SkuDetails)
+
+    /**
+     * Sync subscription state between NewsBlur and Play Store.
+     */
+    fun syncActiveSubscription(): Job
+
+    /**
+     * Notify backend of active Play Store subscription.
+     */
+    fun saveReceipt(purchase: Purchase)
+
+    suspend fun hasActiveSubscription(): Boolean
+}
+
+interface SubscriptionsListener {
+
+    fun onActiveSubscription(renewalMessage: String?)
+
+    fun onAvailableSubscription(skuDetails: SkuDetails)
+
+    fun onBillingConnectionReady()
+
+    fun onBillingConnectionError(message: String? = null)
+}
+
+class SubscriptionManagerImpl(
+        private val context: Context,
+        private val scope: CoroutineScope = NBScope,
+) : SubscriptionManager {
+
+    private var listener: SubscriptionsListener? = null
+
+    private val acknowledgePurchaseListener = AcknowledgePurchaseResponseListener { billingResult: BillingResult ->
+        when (billingResult.responseCode) {
+            BillingClient.BillingResponseCode.OK -> {
+                Log.d(this, "acknowledgePurchaseResponseListener OK")
+                syncActiveSubscription()
+            }
+            BillingClient.BillingResponseCode.BILLING_UNAVAILABLE -> {
+                // Billing API version is not supported for the type requested.
+                Log.d(this, "acknowledgePurchaseResponseListener BILLING_UNAVAILABLE")
+            }
+            BillingClient.BillingResponseCode.SERVICE_UNAVAILABLE -> {
+                // Network connection is down.
+                Log.d(this, "acknowledgePurchaseResponseListener SERVICE_UNAVAILABLE")
+            }
+            else -> {
+                // Handle any other error codes.
+                Log.d(this, "acknowledgePurchaseResponseListener ERROR - message: " + billingResult.debugMessage)
+            }
+        }
+    }
+
+    /**
+     * Billing client listener triggered after every user purchase intent.
+     */
+    private val purchaseUpdateListener = PurchasesUpdatedListener { billingResult: BillingResult, purchases: List<Purchase>? ->
+        if (billingResult.responseCode == BillingClient.BillingResponseCode.OK && purchases != null) {
+            Log.d(this, "purchaseUpdateListener OK")
+            for (purchase in purchases) {
+                handlePurchase(purchase)
+            }
+        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.USER_CANCELED) {
+            // Handle an error caused by a user cancelling the purchase flow.
+            Log.d(this, "purchaseUpdateListener USER_CANCELLED")
+        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.BILLING_UNAVAILABLE) {
+            // Billing API version is not supported for the type requested.
+            Log.d(this, "purchaseUpdateListener BILLING_UNAVAILABLE")
+        } else if (billingResult.responseCode == BillingClient.BillingResponseCode.SERVICE_UNAVAILABLE) {
+            // Network connection is down.
+            Log.d(this, "purchaseUpdateListener SERVICE_UNAVAILABLE")
+        } else {
+            // Handle any other error codes.
+            Log.d(this, "purchaseUpdateListener ERROR - message: " + billingResult.debugMessage)
+        }
+    }
+
+    private val billingClientStateListener: BillingClientStateListener = object : BillingClientStateListener {
+        override fun onBillingSetupFinished(billingResult: BillingResult) {
+            if (billingResult.responseCode == BillingClient.BillingResponseCode.OK) {
+                Log.d(this, "onBillingSetupFinished OK")
+                listener?.onBillingConnectionReady()
+            } else {
+                listener?.onBillingConnectionError("Error connecting to Play Store.")
+            }
+        }
+
+        override fun onBillingServiceDisconnected() {
+            Log.d(this, "onBillingServiceDisconnected")
+            // Try to restart the connection on the next request to
+            // Google Play by calling the startConnection() method.
+            listener?.onBillingConnectionError("Error connecting to Play Store.")
+        }
+    }
+
+    private val billingClient: BillingClient = BillingClient.newBuilder(context)
+            .setListener(purchaseUpdateListener)
+            .enablePendingPurchases()
+            .build()
+
+    override fun startBillingConnection(listener: SubscriptionsListener?) {
+        this.listener = listener
+        billingClient.startConnection(billingClientStateListener)
+    }
+
+    override fun syncSubscriptionState() {
+        scope.launch(Dispatchers.Default) {
+            if (hasActiveSubscription()) syncActiveSubscription()
+            else syncAvailableSubscription()
+        }
+    }
+
+    override fun purchaseSubscription(activity: Activity, skuDetails: SkuDetails) {
+        Log.d(this, "launchBillingFlow for sku: ${skuDetails.sku}")
+        val billingFlowParams = BillingFlowParams.newBuilder()
+                .setSkuDetails(skuDetails)
+                .build()
+        billingClient.launchBillingFlow(activity, billingFlowParams)
+    }
+
+    override fun syncActiveSubscription() = scope.launch(Dispatchers.Default) {
+        val hasNewsBlurSubscription = PrefsUtils.getIsPremium(context)
+        val activePlayStoreSubscription = getActiveSubscriptionAsync().await()
+
+        if (hasNewsBlurSubscription || activePlayStoreSubscription != null) {
+            listener?.let {
+                val renewalString: String? = getRenewalMessage(activePlayStoreSubscription)
+                withContext(Dispatchers.Main) {
+                    it.onActiveSubscription(renewalString)
+                }
+            }
+        }
+
+        if (!hasNewsBlurSubscription && activePlayStoreSubscription != null) {
+            saveReceipt(activePlayStoreSubscription)
+        }
+    }
+
+    override suspend fun hasActiveSubscription(): Boolean =
+            PrefsUtils.getIsPremium(context) || getActiveSubscriptionAsync().await() != null
+
+    override fun saveReceipt(purchase: Purchase) {
+        Log.d(this, "saveReceipt: ${purchase.orderId}")
+        val apiManager = APIManager(context)
+        scope.executeAsyncTask(
+                doInBackground = {
+                    apiManager.saveReceipt(purchase.orderId, purchase.skus.first())
+                },
+                onPostExecute = {
+                    if (!it.isError) {
+                        NBSyncService.forceFeedsFolders()
+                        FeedUtils.triggerSync(context)
+                    }
+                }
+        )
+    }
+
+    private suspend fun syncAvailableSubscription() = scope.launch(Dispatchers.Default) {
+        val skuDetails = getAvailableSubscriptionAsync().await()
+        withContext(Dispatchers.Main) {
+            skuDetails?.let {
+                Log.d(this, it.toString())
+                listener?.onAvailableSubscription(it)
+            } ?: listener?.onBillingConnectionError()
+        }
+    }
+
+    private fun getAvailableSubscriptionAsync(): Deferred<SkuDetails?> {
+        val deferred = CompletableDeferred<SkuDetails?>()
+        val params = SkuDetailsParams.newBuilder().apply {
+            // add subscription SKUs from Play Store
+            setSkusList(listOf(AppConstants.PREMIUM_SKU))
+            setType(BillingClient.SkuType.SUBS)
+        }.build()
+
+        billingClient.querySkuDetailsAsync(params) { _: BillingResult?, skuDetailsList: List<SkuDetails>? ->
+            Log.d(this, "SkuDetailsResponse ${skuDetailsList.toString()}")
+            skuDetailsList?.let {
+                // Currently interested only in the premium yearly News Blur subscription.
+                val skuDetails = it.find { skuDetails ->
+                    skuDetails.sku == AppConstants.PREMIUM_SKU
+                }
+
+                Log.d(this, skuDetails.toString())
+                deferred.complete(skuDetails)
+            } ?: deferred.complete(null)
+        }
+
+        return deferred
+    }
+
+    private fun getActiveSubscriptionAsync(): Deferred<Purchase?> {
+        val deferred = CompletableDeferred<Purchase?>()
+        billingClient.queryPurchasesAsync(BillingClient.SkuType.SUBS) { _, purchases ->
+            val purchase = purchases.find { purchase -> purchase.skus.contains(AppConstants.PREMIUM_SKU) }
+            deferred.complete(purchase)
+        }
+
+        return deferred
+    }
+
+    private fun handlePurchase(purchase: Purchase) {
+        Log.d(this, "handlePurchase: ${purchase.orderId}")
+        if (purchase.purchaseState == Purchase.PurchaseState.PURCHASED && purchase.isAcknowledged) {
+            syncActiveSubscription()
+        } else if (purchase.purchaseState == Purchase.PurchaseState.PURCHASED && !purchase.isAcknowledged) {
+            // need to acknowledge first time sub otherwise it will void
+            Log.d(this, "acknowledge purchase: ${purchase.orderId}")
+            AcknowledgePurchaseParams.newBuilder()
+                    .setPurchaseToken(purchase.purchaseToken)
+                    .build()
+                    .also {
+                        billingClient.acknowledgePurchase(it, acknowledgePurchaseListener)
+                    }
+        }
+    }
+
+    /**
+     * Generate subscription renewal message.
+     */
+    private fun getRenewalMessage(purchase: Purchase?): String? {
+        val expirationTimeMs = PrefsUtils.getPremiumExpire(context)
+        return when {
+            // lifetime subscription
+            expirationTimeMs == 0L -> {
+                context.getString(R.string.premium_subscription_no_expiration)
+            }
+            expirationTimeMs > 0 -> {
+                // date constructor expects ms
+                val expirationDate = Date(expirationTimeMs * 1000)
+                val dateFormat: DateFormat = SimpleDateFormat("EEE, MMMM d, yyyy", Locale.getDefault())
+                dateFormat.timeZone = TimeZone.getDefault()
+                if (purchase != null && !purchase.isAutoRenewing) {
+                    context.getString(R.string.premium_subscription_expiration, dateFormat.format(expirationDate))
+                } else {
+                    context.getString(R.string.premium_subscription_renewal, dateFormat.format(expirationDate))
+                }
+            }
+            else -> null
+        }
+    }
+}

clients/android/NewsBlur/src/com/newsblur/util/AppConstants.java

@@ -41,6 +41,9 @@ public class AppConstants {
     // to account for the fact that it is approximate, and missing a cycle is bad.
     public static final long BG_SERVICE_CYCLE_MILLIS = AUTO_SYNC_TIME_MILLIS + 30L * 1000L;
 
+    // how often to trigger the job scheduler to sync subscription state.
+    public static final long BG_SUBSCRIPTION_SYNC_CYCLE_MILLIS = 24L * 60 * 60 * 1000L;
+
     // how many total attemtps to make at a single API call
     public static final int MAX_API_TRIES = 3;
 

clients/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java

@@ -30,6 +30,7 @@ import com.newsblur.R;
 import com.newsblur.activity.Login;
 import com.newsblur.domain.UserDetails;
 import com.newsblur.network.APIConstants;
+import com.newsblur.service.SubscriptionSyncService;
 import com.newsblur.util.PrefConstants.ThemeValue;
 import com.newsblur.service.NBSyncService;
 import com.newsblur.widget.WidgetUtils;
@@ -169,6 +170,9 @@ public class PrefsUtils {
         NBSyncService.softInterrupt();
         NBSyncService.clearState();
 
+        // cancel scheduled subscription sync service
+        SubscriptionSyncService.cancel(context);
+
         NotificationUtils.clear(context);
 
         // wipe the prefs store
@@ -1026,4 +1030,14 @@ public class PrefsUtils {
         editor.putBoolean(PrefConstants.IN_APP_REVIEW, true);
         editor.commit();
     }
+
+    /**
+     * Check for logged in user.
+     * @return whether a cookie is stored on disk
+     * which gets saved when a user is authenticated.
+     */
+    public static boolean hasCookie(Context context) {
+        SharedPreferences preferences = context.getSharedPreferences(PrefConstants.PREFERENCES, Context.MODE_PRIVATE);
+        return preferences.getString(PrefConstants.PREF_COOKIE, null) != null;
+    }
 }

clients/android/NewsBlur/src/com/newsblur/viewModel/AllFoldersViewModel.kt

@@ -0,0 +1,71 @@
+package com.newsblur.viewModel
+
+import android.database.Cursor
+import android.os.CancellationSignal
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MutableLiveData
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import com.newsblur.util.FeedUtils
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+class AllFoldersViewModel : ViewModel() {
+
+    private val cancellationSignal = CancellationSignal()
+
+    // social feeds
+    private val _socialFeeds = MutableLiveData<Cursor>()
+    val socialFeeds: LiveData<Cursor> = _socialFeeds
+
+    // folders
+    private val _folders = MutableLiveData<Cursor>()
+    val folders: LiveData<Cursor> = _folders
+
+    // feeds
+    private val _feeds = MutableLiveData<Cursor>()
+    val feeds: LiveData<Cursor> = _feeds
+
+    // saved story counts
+    private val _savedStoryCounts = MutableLiveData<Cursor>()
+    val savedStoryCounts: LiveData<Cursor> = _savedStoryCounts
+
+    // saved search
+    private val _savedSearch = MutableLiveData<Cursor>()
+    val savedSearch: LiveData<Cursor> = _savedSearch
+
+    fun getData() {
+        viewModelScope.launch(Dispatchers.IO) {
+            launch {
+                FeedUtils.dbHelper!!.getSocialFeedsCursor(cancellationSignal).let {
+                    _socialFeeds.postValue(it)
+                }
+            }
+            launch {
+                FeedUtils.dbHelper!!.getFoldersCursor(cancellationSignal).let {
+                    _folders.postValue(it)
+                }
+            }
+            launch {
+                FeedUtils.dbHelper!!.getFeedsCursor(cancellationSignal).let {
+                    _feeds.postValue(it)
+                }
+            }
+            launch {
+                FeedUtils.dbHelper!!.getSavedStoryCountsCursor(cancellationSignal).let {
+                    _savedStoryCounts.postValue(it)
+                }
+            }
+            launch {
+                FeedUtils.dbHelper!!.getSavedSearchCursor(cancellationSignal).let {
+                    _savedSearch.postValue(it)
+                }
+            }
+        }
+    }
+
+    override fun onCleared() {
+        cancellationSignal.cancel()
+        super.onCleared()
+    }
+}

clients/android/NewsBlur/src/com/newsblur/viewModel/FeedFolderViewModel.kt

@@ -0,0 +1,49 @@
+package com.newsblur.viewModel
+
+import android.database.Cursor
+import android.os.CancellationSignal
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MutableLiveData
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import com.newsblur.util.FeedUtils
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+class FeedFolderViewModel : ViewModel() {
+
+    private val cancellationSignal = CancellationSignal()
+
+    private val _folders = MutableLiveData<Cursor>()
+    val foldersLiveData: LiveData<Cursor> = _folders
+    private val _feeds = MutableLiveData<Cursor>()
+    val feedsLiveData: LiveData<Cursor> = _feeds
+
+    fun getData() {
+        viewModelScope.launch(Dispatchers.IO) {
+            launch {
+                FeedUtils.dbHelper!!.getFoldersCursor(cancellationSignal).let {
+                    _folders.postValue(it)
+                }
+            }
+            launch {
+                FeedUtils.dbHelper!!.getFeedsCursor(cancellationSignal).let {
+                    _feeds.postValue(it)
+                }
+            }
+        }
+    }
+
+    fun getFeeds() {
+        viewModelScope.launch(Dispatchers.IO) {
+            FeedUtils.dbHelper!!.getFeedsCursor(cancellationSignal).let {
+                _feeds.postValue(it)
+            }
+        }
+    }
+
+    override fun onCleared() {
+        cancellationSignal.cancel()
+        super.onCleared()
+    }
+}

clients/android/NewsBlur/src/com/newsblur/viewModel/StoriesViewModel.kt

@@ -0,0 +1,32 @@
+package com.newsblur.viewModel
+
+import android.database.Cursor
+import android.os.CancellationSignal
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MutableLiveData
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import com.newsblur.util.FeedSet
+import com.newsblur.util.FeedUtils
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+class StoriesViewModel : ViewModel() {
+
+    private val cancellationSignal = CancellationSignal()
+    private val _activeStoriesLiveData = MutableLiveData<Cursor>()
+    val activeStoriesLiveData: LiveData<Cursor> = _activeStoriesLiveData
+
+    fun getActiveStories(fs: FeedSet) {
+        viewModelScope.launch(Dispatchers.IO) {
+            FeedUtils.dbHelper!!.getActiveStoriesCursor(fs, cancellationSignal).let {
+                _activeStoriesLiveData.postValue(it)
+            }
+        }
+    }
+
+    override fun onCleared() {
+        cancellationSignal.cancel()
+        super.onCleared()
+    }
+}

clients/android/NewsBlur/src/com/newsblur/viewModel/StoryUserTagsViewModel.kt

@@ -0,0 +1,30 @@
+package com.newsblur.viewModel
+
+import android.database.Cursor
+import android.os.CancellationSignal
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MutableLiveData
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import com.newsblur.util.FeedUtils
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+
+class StoryUserTagsViewModel : ViewModel() {
+
+    private val cancellationSignal = CancellationSignal()
+    private val _savedStoryCountsLiveData = MutableLiveData<Cursor>()
+    val savedStoryCountsLiveData: LiveData<Cursor> = _savedStoryCountsLiveData
+
+    fun getSavedStoryCounts() {
+        viewModelScope.launch(Dispatchers.IO) {
+            val cursor = FeedUtils.dbHelper!!.getSavedStoryCountsCursor(cancellationSignal)
+            _savedStoryCountsLiveData.postValue(cursor)
+        }
+    }
+
+    override fun onCleared() {
+        cancellationSignal.cancel()
+        super.onCleared()
+    }
+}

clients/android/NewsBlur/src/com/newsblur/widget/WidgetRemoteViewsFactory.java

@@ -1,270 +0,0 @@
-package com.newsblur.widget;
-
-import android.appwidget.AppWidgetManager;
-import android.content.Context;
-import android.content.Intent;
-import android.database.Cursor;
-import android.graphics.Color;
-import android.os.Bundle;
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-import androidx.loader.content.Loader;
-import android.text.TextUtils;
-import android.view.View;
-import android.widget.RemoteViews;
-import android.widget.RemoteViewsService;
-
-import com.newsblur.R;
-import com.newsblur.domain.Feed;
-import com.newsblur.domain.Story;
-import com.newsblur.network.APIManager;
-import com.newsblur.network.domain.StoriesResponse;
-import com.newsblur.util.FeedSet;
-import com.newsblur.util.FeedUtils;
-import com.newsblur.util.Log;
-import com.newsblur.util.PrefsUtils;
-import com.newsblur.util.ReadFilter;
-import com.newsblur.util.StoryOrder;
-import com.newsblur.util.StoryUtils;
-import com.newsblur.util.ThumbnailStyle;
-import com.newsblur.util.UIUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Set;
-
-public class WidgetRemoteViewsFactory implements RemoteViewsService.RemoteViewsFactory {
-
-    private static String TAG = "WidgetRemoteViewsFactory";
-
-    private Context context;
-    private APIManager apiManager;
-    private List<Story> storyItems = new ArrayList<>();
-    private FeedSet fs;
-    private int appWidgetId;
-    private boolean dataCompleted;
-
-    WidgetRemoteViewsFactory(Context context, Intent intent) {
-        com.newsblur.util.Log.d(TAG, "Constructor");
-        this.context = context;
-        appWidgetId = intent.getIntExtra(AppWidgetManager.EXTRA_APPWIDGET_ID,
-                AppWidgetManager.INVALID_APPWIDGET_ID);
-    }
-
-    /**
-     * The system calls onCreate() when creating your factory for the first time.
-     * This is where you set up any connections and/or cursors to your data source.
-     * <p>
-     * Heavy lifting,
-     * for example downloading or creating content etc, should be deferred to onDataSetChanged()
-     * or getViewAt(). Taking more than 20 seconds in this call will result in an ANR.
-     */
-    @Override
-    public void onCreate() {
-        Log.d(TAG, "onCreate");
-        this.apiManager = new APIManager(context);
-        // widget could be created before app init
-        // wait for the dbHelper to be ready for use
-        while (FeedUtils.dbHelper == null) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-            if (FeedUtils.dbHelper == null) {
-                FeedUtils.offerInitContext(context);
-            }
-        }
-
-        WidgetUtils.enableWidgetUpdate(context);
-    }
-
-    /**
-     * Allowed to run synchronous calls
-     */
-    @Override
-    public RemoteViews getViewAt(int position) {
-        com.newsblur.util.Log.d(TAG, "getViewAt " + position);
-        Story story = storyItems.get(position);
-
-        WidgetRemoteViews rv = new WidgetRemoteViews(context.getPackageName(), R.layout.view_widget_story_item);
-        rv.setTextViewText(R.id.story_item_title, story.title);
-        rv.setTextViewText(R.id.story_item_content, story.shortContent);
-        rv.setTextViewText(R.id.story_item_author, story.authors);
-        rv.setTextViewText(R.id.story_item_feedtitle, story.extern_feedTitle);
-        CharSequence time = StoryUtils.formatShortDate(context, story.timestamp);
-        rv.setTextViewText(R.id.story_item_date, time);
-
-        // image dimensions same as R.layout.view_widget_story_item
-        FeedUtils.iconLoader.displayWidgetImage(story.extern_faviconUrl, R.id.story_item_feedicon, UIUtils.dp2px(context, 19), rv);
-        if (PrefsUtils.getThumbnailStyle(context) != ThumbnailStyle.OFF && !TextUtils.isEmpty(story.thumbnailUrl)) {
-            FeedUtils.thumbnailLoader.displayWidgetImage(story.thumbnailUrl, R.id.story_item_thumbnail, UIUtils.dp2px(context, 64), rv);
-        } else {
-            rv.setViewVisibility(R.id.story_item_thumbnail, View.GONE);
-        }
-
-        rv.setViewBackgroundColor(R.id.story_item_favicon_borderbar_1, UIUtils.decodeColourValue(story.extern_feedColor, Color.GRAY));
-        rv.setViewBackgroundColor(R.id.story_item_favicon_borderbar_2, UIUtils.decodeColourValue(story.extern_feedFade, Color.LTGRAY));
-
-        // set fill-intent which is used to fill in the pending intent template
-        // set on the collection view in WidgetProvider
-        Bundle extras = new Bundle();
-        extras.putString(WidgetUtils.EXTRA_ITEM_ID, story.storyHash);
-        Intent fillInIntent = new Intent();
-        fillInIntent.putExtras(extras);
-
-        rv.setOnClickFillInIntent(R.id.view_widget_item, fillInIntent);
-        return rv;
-    }
-
-    /**
-     * This allows for the use of a custom loading view which appears between the time that
-     * {@link #getViewAt(int)} is called and returns. If null is returned, a default loading
-     * view will be used.
-     *
-     * @return The RemoteViews representing the desired loading view.
-     */
-    @Override
-    public RemoteViews getLoadingView() {
-        return null;
-    }
-
-    /**
-     * @return The number of types of Views that will be returned by this factory.
-     */
-    @Override
-    public int getViewTypeCount() {
-        return 1;
-    }
-
-    /**
-     * @param position The position of the item within the data set whose row id we want.
-     * @return The id of the item at the specified position.
-     */
-    @Override
-    public long getItemId(int position) {
-        return storyItems.get(position).hashCode();
-    }
-
-    /**
-     * @return True if the same id always refers to the same object.
-     */
-    @Override
-    public boolean hasStableIds() {
-        return true;
-    }
-
-    @Override
-    public void onDataSetChanged() {
-        com.newsblur.util.Log.d(TAG, "onDataSetChanged");
-        // if user logged out don't try to update widget
-        if (!WidgetUtils.isLoggedIn(context)) {
-            com.newsblur.util.Log.d(TAG, "onDataSetChanged - not logged in");
-            return;
-        }
-
-        if (dataCompleted) {
-            // we have all the stories data, just let the widget redraw
-            com.newsblur.util.Log.d(TAG, "onDataSetChanged - redraw widget");
-            dataCompleted = false;
-        } else {
-            setFeedSet();
-            if (fs == null) {
-                com.newsblur.util.Log.d(TAG, "onDataSetChanged - null feed set. Show empty view");
-                setStories(new Story[]{}, new HashMap<String, Feed>(0));
-                return;
-            }
-
-            com.newsblur.util.Log.d(TAG, "onDataSetChanged - fetch stories");
-            StoriesResponse response = apiManager.getStories(fs, 1, StoryOrder.NEWEST, ReadFilter.ALL);
-
-            if (response == null || response.stories == null) {
-                com.newsblur.util.Log.d(TAG, "Error fetching widget stories");
-            } else {
-                com.newsblur.util.Log.d(TAG, "Fetched widget stories");
-                processStories(response.stories);
-                FeedUtils.dbHelper.insertStories(response, true);
-            }
-        }
-    }
-
-    /**
-     * Called when the last RemoteViewsAdapter that is associated with this factory is
-     * unbound.
-     */
-    @Override
-    public void onDestroy() {
-        com.newsblur.util.Log.d(TAG, "onDestroy");
-        WidgetUtils.disableWidgetUpdate(context);
-        PrefsUtils.removeWidgetData(context);
-    }
-
-    /**
-     * @return Count of items.
-     */
-    @Override
-    public int getCount() {
-        return Math.min(storyItems.size(), WidgetUtils.STORIES_LIMIT);
-    }
-
-    private void processStories(final Story[] stories) {
-        com.newsblur.util.Log.d(TAG, "processStories");
-        final HashMap<String, Feed> feedMap = new HashMap<>();
-        Loader<Cursor> loader = FeedUtils.dbHelper.getFeedsLoader();
-        loader.registerListener(loader.getId(), new Loader.OnLoadCompleteListener<Cursor>() {
-            @Override
-            public void onLoadComplete(@NonNull Loader<Cursor> loader, @Nullable Cursor cursor) {
-                while (cursor != null && cursor.moveToNext()) {
-                    Feed feed = Feed.fromCursor(cursor);
-                    if (feed.active) {
-                        feedMap.put(feed.feedId, feed);
-                    }
-                }
-                setStories(stories, feedMap);
-            }
-        });
-        loader.startLoading();
-    }
-
-    private void setStories(Story[] stories, HashMap<String, Feed> feedMap) {
-        com.newsblur.util.Log.d(TAG, "setStories");
-        for (Story story : stories) {
-            Feed storyFeed = feedMap.get(story.feedId);
-            if (storyFeed != null) {
-                bindStoryValues(story, storyFeed);
-            }
-        }
-        this.storyItems.clear();
-        this.storyItems.addAll(Arrays.asList(stories));
-        // we have the data, notify data set changed
-        dataCompleted = true;
-        invalidate();
-    }
-
-    private void bindStoryValues(Story story, Feed feed) {
-        story.thumbnailUrl = Story.guessStoryThumbnailURL(story);
-        story.extern_faviconBorderColor = feed.faviconBorder;
-        story.extern_faviconUrl = feed.faviconUrl;
-        story.extern_feedTitle = feed.title;
-        story.extern_feedFade = feed.faviconFade;
-        story.extern_feedColor = feed.faviconColor;
-    }
-
-    private void invalidate() {
-        com.newsblur.util.Log.d(TAG, "Invalidate app widget with id: " + appWidgetId);
-        AppWidgetManager appWidgetManager = AppWidgetManager.getInstance(context);
-        appWidgetManager.notifyAppWidgetViewDataChanged(appWidgetId, R.id.widget_list);
-    }
-
-    private void setFeedSet() {
-        Set<String> feedIds = PrefsUtils.getWidgetFeedIds(context);
-        if (feedIds == null || !feedIds.isEmpty()) {
-            fs = FeedSet.widgetFeeds(feedIds);
-        } else {
-            // no feeds selected. Widget will show tap to config view
-            fs = null;
-        }
-    }
-}

clients/android/NewsBlur/src/com/newsblur/widget/WidgetRemoteViewsFactory.kt

@@ -0,0 +1,231 @@
+package com.newsblur.widget
+
+import android.appwidget.AppWidgetManager
+import android.content.Context
+import android.content.Intent
+import android.graphics.Color
+import android.os.Bundle
+import android.os.CancellationSignal
+import android.text.TextUtils
+import android.view.View
+import android.widget.RemoteViews
+import android.widget.RemoteViewsService.RemoteViewsFactory
+import com.newsblur.R
+import com.newsblur.domain.Feed
+import com.newsblur.domain.Story
+import com.newsblur.network.APIManager
+import com.newsblur.util.*
+import com.newsblur.util.FeedUtils.offerInitContext
+import java.util.*
+import kotlin.math.min
+
+class WidgetRemoteViewsFactory internal constructor(context: Context, intent: Intent) : RemoteViewsFactory {
+
+    private val context: Context
+    private var fs: FeedSet? = null
+    private val appWidgetId: Int
+    private var dataCompleted = false
+    private val storyItems: MutableList<Story> = ArrayList()
+    private val cancellationSignal = CancellationSignal()
+    private var apiManager: APIManager? = null
+
+    init {
+        Log.d(TAG, "Constructor")
+        this.context = context
+        appWidgetId = intent.getIntExtra(AppWidgetManager.EXTRA_APPWIDGET_ID,
+                AppWidgetManager.INVALID_APPWIDGET_ID)
+    }
+
+    /**
+     * The system calls onCreate() when creating your factory for the first time.
+     * This is where you set up any connections and/or cursors to your data source.
+     *
+     *
+     * Heavy lifting,
+     * for example downloading or creating content etc, should be deferred to onDataSetChanged()
+     * or getViewAt(). Taking more than 20 seconds in this call will result in an ANR.
+     */
+    override fun onCreate() {
+        Log.d(TAG, "onCreate")
+        apiManager = APIManager(context)
+        // widget could be created before app init
+        // wait for the dbHelper to be ready for use
+        while (FeedUtils.dbHelper == null) {
+            try {
+                Thread.sleep(500)
+            } catch (e: InterruptedException) {
+                e.printStackTrace()
+            }
+            if (FeedUtils.dbHelper == null) {
+                offerInitContext(context)
+            }
+        }
+        WidgetUtils.enableWidgetUpdate(context)
+    }
+
+    /**
+     * Allowed to run synchronous calls
+     */
+    override fun getViewAt(position: Int): RemoteViews {
+        Log.d(TAG, "getViewAt $position")
+        val story = storyItems[position]
+        val rv = WidgetRemoteViews(context.packageName, R.layout.view_widget_story_item)
+        rv.setTextViewText(R.id.story_item_title, story.title)
+        rv.setTextViewText(R.id.story_item_content, story.shortContent)
+        rv.setTextViewText(R.id.story_item_author, story.authors)
+        rv.setTextViewText(R.id.story_item_feedtitle, story.extern_feedTitle)
+        val time: CharSequence = StoryUtils.formatShortDate(context, story.timestamp)
+        rv.setTextViewText(R.id.story_item_date, time)
+
+        // image dimensions same as R.layout.view_widget_story_item
+        FeedUtils.iconLoader!!.displayWidgetImage(story.extern_faviconUrl, R.id.story_item_feedicon, UIUtils.dp2px(context, 19), rv)
+        if (PrefsUtils.getThumbnailStyle(context) != ThumbnailStyle.OFF && !TextUtils.isEmpty(story.thumbnailUrl)) {
+            FeedUtils.thumbnailLoader!!.displayWidgetImage(story.thumbnailUrl, R.id.story_item_thumbnail, UIUtils.dp2px(context, 64), rv)
+        } else {
+            rv.setViewVisibility(R.id.story_item_thumbnail, View.GONE)
+        }
+        rv.setViewBackgroundColor(R.id.story_item_favicon_borderbar_1, UIUtils.decodeColourValue(story.extern_feedColor, Color.GRAY))
+        rv.setViewBackgroundColor(R.id.story_item_favicon_borderbar_2, UIUtils.decodeColourValue(story.extern_feedFade, Color.LTGRAY))
+
+        // set fill-intent which is used to fill in the pending intent template
+        // set on the collection view in WidgetProvider
+        val extras = Bundle()
+        extras.putString(WidgetUtils.EXTRA_ITEM_ID, story.storyHash)
+        val fillInIntent = Intent()
+        fillInIntent.putExtras(extras)
+        rv.setOnClickFillInIntent(R.id.view_widget_item, fillInIntent)
+        return rv
+    }
+
+    /**
+     * This allows for the use of a custom loading view which appears between the time that
+     * [.getViewAt] is called and returns. If null is returned, a default loading
+     * view will be used.
+     *
+     * @return The RemoteViews representing the desired loading view.
+     */
+    override fun getLoadingView(): RemoteViews? = null
+
+    /**
+     * @return The number of types of Views that will be returned by this factory.
+     */
+    override fun getViewTypeCount(): Int = 1
+
+    /**
+     * @param position The position of the item within the data set whose row id we want.
+     * @return The id of the item at the specified position.
+     */
+    override fun getItemId(position: Int): Long = storyItems[position].hashCode().toLong()
+
+    /**
+     * @return True if the same id always refers to the same object.
+     */
+    override fun hasStableIds(): Boolean = true
+
+    override fun onDataSetChanged() {
+        Log.d(TAG, "onDataSetChanged")
+        // if user logged out don't try to update widget
+        if (!WidgetUtils.isLoggedIn(context)) {
+            Log.d(TAG, "onDataSetChanged - not logged in")
+            return
+        }
+        if (dataCompleted) {
+            // we have all the stories data, just let the widget redraw
+            Log.d(TAG, "onDataSetChanged - redraw widget")
+            dataCompleted = false
+        } else {
+            setFeedSet()
+            if (fs == null) {
+                Log.d(TAG, "onDataSetChanged - null feed set. Show empty view")
+                setStories(arrayOf(), HashMap(0))
+                return
+            }
+            Log.d(TAG, "onDataSetChanged - fetch stories")
+            val response = apiManager!!.getStories(fs, 1, StoryOrder.NEWEST, ReadFilter.ALL)
+            if (response?.stories == null) {
+                Log.d(TAG, "Error fetching widget stories")
+            } else {
+                Log.d(TAG, "Fetched widget stories")
+                processStories(response.stories)
+                FeedUtils.dbHelper!!.insertStories(response, true)
+            }
+        }
+    }
+
+    /**
+     * Called when the last RemoteViewsAdapter that is associated with this factory is
+     * unbound.
+     */
+    override fun onDestroy() {
+        Log.d(TAG, "onDestroy")
+        cancellationSignal.cancel()
+        WidgetUtils.disableWidgetUpdate(context)
+        PrefsUtils.removeWidgetData(context)
+    }
+
+    /**
+     * @return Count of items.
+     */
+    override fun getCount(): Int = min(storyItems.size, WidgetUtils.STORIES_LIMIT)
+
+    private fun processStories(stories: Array<Story>) {
+        Log.d(TAG, "processStories")
+        val feedMap = HashMap<String, Feed>()
+        NBScope.executeAsyncTask(
+                doInBackground = {
+                    FeedUtils.dbHelper!!.getFeedsCursor(cancellationSignal)
+                },
+                onPostExecute = {
+                    while (it != null && it.moveToNext()) {
+                        val feed = Feed.fromCursor(it)
+                        if (feed.active) {
+                            feedMap[feed.feedId] = feed
+                        }
+                    }
+                    setStories(stories, feedMap)
+                }
+        )
+    }
+
+    private fun setStories(stories: Array<Story>, feedMap: HashMap<String, Feed>) {
+        Log.d(TAG, "setStories")
+        for (story in stories) {
+            val storyFeed = feedMap[story.feedId]
+            storyFeed?.let { bindStoryValues(story, it) }
+        }
+        storyItems.clear()
+        storyItems.addAll(mutableListOf(*stories))
+        // we have the data, notify data set changed
+        dataCompleted = true
+        invalidate()
+    }
+
+    private fun bindStoryValues(story: Story, feed: Feed) {
+        story.thumbnailUrl = Story.guessStoryThumbnailURL(story)
+        story.extern_faviconBorderColor = feed.faviconBorder
+        story.extern_faviconUrl = feed.faviconUrl
+        story.extern_feedTitle = feed.title
+        story.extern_feedFade = feed.faviconFade
+        story.extern_feedColor = feed.faviconColor
+    }
+
+    private fun invalidate() {
+        Log.d(TAG, "Invalidate app widget with id: $appWidgetId")
+        val appWidgetManager = AppWidgetManager.getInstance(context)
+        appWidgetManager.notifyAppWidgetViewDataChanged(appWidgetId, R.id.widget_list)
+    }
+
+    private fun setFeedSet() {
+        val feedIds = PrefsUtils.getWidgetFeedIds(context)
+        fs = if (feedIds == null || feedIds.isNotEmpty()) {
+            FeedSet.widgetFeeds(feedIds)
+        } else {
+            // no feeds selected. Widget will show tap to config view
+            null
+        }
+    }
+
+    companion object {
+        private const val TAG = "WidgetRemoteViewsFactory"
+    }
+}

config/requirements.txt

@@ -1,5 +1,4 @@
 amqp==2.6.1
-anyjson==0.3.3
 apns2==0.7.2
 appdirs==1.4.4
 asgiref==3.3.4
@@ -27,6 +26,7 @@ django-nose==1.4.7
 django-oauth-toolkit==1.3.3
 django-paypal==1.0.0
 django-qurl==0.1.1
+django-pipeline>=2,<3
 django-redis-cache==3.0.0
 django-redis-sessions==0.6.1
 django-ses==1.0.3
@@ -78,7 +78,7 @@ pbr==5.6.0
 Pillow==8.0.1
 pluggy==0.13.1
 psutil==5.7.3
-psycopg2==2.8.6
+psycopg2==2.9.2
 py==1.10.0
 pyasn1==0.4.8
 pycparser==2.20

docker-compose.metrics.yml

@@ -15,10 +15,13 @@ services:
       - node-exporter
       - haproxy
       - flask_metrics_mongo
+      - flask_metrics_redis
+      - flask_metrics_haproxy
     external_links:
       - haproxy
       - flask_metrics_mongo
       - flask_metrics_redis
+      - flask_metrics_haproxy
 
   node-exporter:
     container_name: node-exporter
@@ -41,6 +44,7 @@ services:
       - ./docker/grafana/dashboards/:/etc/grafana/provisioning/dashboards/
     external_links:
       - prometheus
+  
   flask_metrics_mongo:
     container_name: flask_metrics_mongo
     image: newsblur/newsblur_monitor:latest
@@ -55,6 +59,7 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
+  
   flask_metrics_redis:
     container_name: flask_metrics_redis
     image: newsblur/newsblur_monitor:latest
@@ -69,6 +74,20 @@ services:
       - nginx
     volumes:
       - ${PWD}:/srv/newsblur
+
+  flask_metrics_haproxy:
+    container_name: flask_metrics_haproxy
+    image: newsblur/newsblur_monitor:latest
+    command: bash -c "python /srv/newsblur/flask_metrics/flask_metrics_haproxy.py"
+    environment: 
+      - DOCKERBUILD=True
+    ports:
+      - 5599:5569
+    depends_on: 
+      - haproxy
+    volumes:
+      - ${PWD}:/srv/newsblur
+  
   elasticsearch_exporter:
     container_name: elasticsearch_exporter
     image: prometheuscommunity/elasticsearch-exporter:latest
@@ -85,4 +104,4 @@ services:
     environment:
       DATA_SOURCE_NAME: 'postgresql://newsblur:newsblur@db_postgres:5432/postgres?sslmode=disable'
     ports:
-    - '9187:9187'
+    - '9187:9187'

docker-compose.yml

@@ -5,6 +5,7 @@ services:
     hostname: nb.com
     container_name: newsblur_web
     image: newsblur/newsblur_${NEWSBLUR_BASE:-python3}:latest
+    user: "${CURRENT_UID}:${CURRENT_GID}"
     environment: 
       - DOCKERBUILD=True
       - RUNWITHMAKEBUILD=${RUNWITHMAKEBUILD?Use the `make` command instead of docker CLI}
@@ -29,8 +30,9 @@ services:
       - ${PWD}:/srv/newsblur
 
   newsblur_node:
-    image: newsblur/newsblur_node:latest
     container_name: node
+    image: newsblur/newsblur_node:latest
+    user: "${CURRENT_UID}:${CURRENT_GID}"
     environment:
       - NODE_ENV=docker
       - MONGODB_PORT=29019
@@ -47,8 +49,9 @@ services:
       - ${PWD}/node:/srv
   
   imageproxy:
-    image: willnorris/imageproxy:latest
     container_name: imageproxy
+    image: ghcr.io/willnorris/imageproxy:latest
+    user: "${CURRENT_UID}:${CURRENT_GID}"
     entrypoint: /app/imageproxy -addr 0.0.0.0:8088 -cache /tmp/imageproxy -verbose
     restart: unless-stopped
     ports:
@@ -93,20 +96,20 @@ services:
       - ./docker/volumes/postgres:/var/lib/postgresql/data
 
   db_redis:
+    container_name: db_redis
     image: redis:latest
     ports:
       - 6579:6579
-    container_name: db_redis
     restart: unless-stopped
     volumes:
-      - ./config/redis.conf:/etc/redis/redis.conf
-      - ./config/redis_docker.conf:/etc/redis/redis_server.conf
-      - ./docker/volumes/redis:/var/lib/redis
+      - ./docker/redis/redis.conf:/etc/redis/redis.conf
+      - ./docker/redis/redis_server.conf:/usr/local/etc/redis/redis_replica.conf
+      - ./docker/volumes/redis:/data
     command: redis-server /etc/redis/redis.conf --port 6579
 
   db_elasticsearch:
     container_name: db_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.11.1
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
     mem_limit: 512mb
     restart: unless-stopped
     environment:
@@ -131,13 +134,13 @@ services:
   task_celery:
     container_name: task_celery
     image: newsblur/newsblur_python3
+    user: "${CURRENT_UID}:${CURRENT_GID}"
     command: "celery worker -A newsblur_web -B --loglevel=INFO"
     restart: unless-stopped
     volumes:
       - ${PWD}:/srv/newsblur
     environment: 
       - DOCKERBUILD=True
-    user: "${CURRENT_UID}:${CURRENT_GID}"
 
   haproxy:
     container_name: haproxy

docker/grafana/dashboards/dashboards.yaml

@@ -11,13 +11,13 @@ providers:
     allowUiUpdates: true
     type: file
     options:
-      path: /etc/grafana/provisioning/dashboards/node_exporter_dashboard.json
+      path: /etc/grafana/provisioning/dashboards/node-exporter_dashboard.json
       foldersFromFilesStructure: true
   - name: MongoDB
     allowUiUpdates: true
     type: file
     options:
-      path: /etc/grafana/provisioning/dashboards/mongo_dashboard.json
+      path: /etc/grafana/provisioning/dashboards/mongodb_dashboard.json
       foldersFromFilesStructure: true
   - name: Redis
     allowUiUpdates: true

docker/grafana/dashboards/mongo_dashboard.json

@@ -1,554 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "gnetId": null,
-  "graphTooltip": 0,
-  "id": 7,
-  "iteration": 1639165228171,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "unit": "short"
-        },
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "hiddenSeries": false,
-      "id": 10,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "mongo_page_queues{instance=\"$instance\"}",
-          "interval": "",
-          "legendFormat": "{{ type }}",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Page Queues",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:978",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:979",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "unit": "decbytes"
-        },
-        "overrides": []
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "hiddenSeries": false,
-      "id": 4,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "mongo_db_size{instance=\"$instance\"}",
-          "interval": "",
-          "legendFormat": "DB Size in Bytes",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Size",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1056",
-          "format": "decbytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:1057",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 8
-      },
-      "hiddenSeries": false,
-      "id": 8,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "rate(mongo_page_faults{instance=\"$instance\"}[6h])",
-          "interval": "",
-          "legendFormat": "Page Faults",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Page Faults",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1212",
-          "decimals": null,
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": ".01",
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:1213",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 8
-      },
-      "hiddenSeries": false,
-      "id": 2,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "mongo_objects{instance=~\"$instance\"}",
-          "interval": "",
-          "legendFormat": "Number of Objects",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Objects",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 16
-      },
-      "hiddenSeries": false,
-      "id": 6,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "mongo_ops{instance=\"$instance\"}",
-          "interval": "",
-          "legendFormat": "{{ type }}",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Ops",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1134",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:1135",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    }
-  ],
-  "refresh": false,
-  "schemaVersion": 32,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "allValue": null,
-        "current": {
-          "selected": false,
-          "text": "db-mongo-primary1",
-          "value": "db-mongo-primary1"
-        },
-        "datasource": "Prometheus",
-        "definition": "label_values(mongo_objects, instance)",
-        "description": null,
-        "error": null,
-        "hide": 0,
-        "includeAll": false,
-        "label": null,
-        "multi": false,
-        "name": "instance",
-        "options": [],
-        "query": {
-          "query": "label_values(mongo_objects, instance)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      }
-    ]
-  },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "MongoDB",
-  "uid": "y5QlZvM7k",
-  "version": 36
-}

docker/grafana/dashboards/mongodb_dashboard.json

@@ -0,0 +1,554 @@
+{
+    "annotations": {
+        "list": [
+            {
+                "builtIn": 1,
+                "datasource": "-- Grafana --",
+                "enable": true,
+                "hide": true,
+                "iconColor": "rgba(0, 211, 255, 1)",
+                "name": "Annotations & Alerts",
+                "target": {
+                    "limit": 100,
+                    "matchAny": false,
+                    "tags": [],
+                    "type": "dashboard"
+                },
+                "type": "dashboard"
+            }
+        ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "id": 7,
+    "iteration": 1639165228171,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fieldConfig": {
+                "defaults": {
+                    "unit": "short"
+                },
+                "overrides": []
+            },
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 0
+            },
+            "hiddenSeries": false,
+            "id": 10,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "mongo_page_queues{instance=\"$instance\"}",
+                    "interval": "",
+                    "legendFormat": "{{ type }}",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Page Queues",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:978",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:979",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fieldConfig": {
+                "defaults": {
+                    "unit": "decbytes"
+                },
+                "overrides": []
+            },
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 0
+            },
+            "hiddenSeries": false,
+            "id": 4,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "mongo_db_size{instance=\"$instance\"}",
+                    "interval": "",
+                    "legendFormat": "DB Size in Bytes",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Size",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:1056",
+                    "format": "decbytes",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:1057",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 8
+            },
+            "hiddenSeries": false,
+            "id": 8,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "rate(mongo_page_faults{instance=\"$instance\"}[6h])",
+                    "interval": "",
+                    "legendFormat": "Page Faults",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Page Faults",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:1212",
+                    "decimals": null,
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": ".01",
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:1213",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 9,
+                "w": 12,
+                "x": 12,
+                "y": 8
+            },
+            "hiddenSeries": false,
+            "id": 2,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "mongo_objects{instance=~\"$instance\"}",
+                    "interval": "",
+                    "legendFormat": "Number of Objects",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Objects",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 16
+            },
+            "hiddenSeries": false,
+            "id": 6,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "mongo_ops{instance=\"$instance\"}",
+                    "interval": "",
+                    "legendFormat": "{{ type }}",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Ops",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:1134",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:1135",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        }
+    ],
+    "refresh": false,
+    "schemaVersion": 32,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+        "list": [
+            {
+                "allValue": null,
+                "current": {
+                    "selected": false,
+                    "text": "db-mongo-primary1",
+                    "value": "db-mongo-primary1"
+                },
+                "datasource": "Prometheus",
+                "definition": "label_values(mongo_objects, instance)",
+                "description": null,
+                "error": null,
+                "hide": 0,
+                "includeAll": false,
+                "label": null,
+                "multi": false,
+                "name": "instance",
+                "options": [],
+                "query": {
+                    "query": "label_values(mongo_objects, instance)",
+                    "refId": "StandardVariableQuery"
+                },
+                "refresh": 1,
+                "regex": "",
+                "skipUrlSync": false,
+                "sort": 0,
+                "tagValuesQuery": "",
+                "tagsQuery": "",
+                "type": "query",
+                "useTags": false
+            }
+        ]
+    },
+    "time": {
+        "from": "now-6h",
+        "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "MongoDB",
+    "uid": "y5QlZvM7k",
+    "version": 37
+}

docker/grafana/dashboards/newsblur_dashboard.json

@@ -26,7 +26,7 @@
   "fiscalYearStartMonth": 0,
   "gnetId": null,
   "graphTooltip": 1,
-  "id": 2,
+  "id": 4,
   "links": [
     {
       "asDropdown": false,
@@ -43,271 +43,6 @@
   ],
   "liveNow": false,
   "panels": [
-    {
-      "datasource": "Prometheus",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "fillOpacity": 70,
-            "lineWidth": 0
-          },
-          "mappings": [
-            {
-              "options": {
-                "0": {
-                  "color": "super-light-red",
-                  "index": 0,
-                  "text": "UNK"
-                },
-                "1": {
-                  "color": "semi-dark-yellow",
-                  "index": 1,
-                  "text": "INI"
-                },
-                "2": {
-                  "color": "semi-dark-red",
-                  "index": 2,
-                  "text": "SOCKERR"
-                },
-                "3": {
-                  "color": "dark-green",
-                  "index": 3,
-                  "text": "L4OK"
-                },
-                "4": {
-                  "color": "dark-red",
-                  "index": 4,
-                  "text": "L4TOUT"
-                },
-                "5": {
-                  "color": "dark-red",
-                  "index": 5,
-                  "text": "L4CON"
-                },
-                "6": {
-                  "color": "dark-green",
-                  "index": 6,
-                  "text": "L6OK"
-                },
-                "7": {
-                  "color": "dark-red",
-                  "index": 7,
-                  "text": "L6TOUT"
-                },
-                "8": {
-                  "color": "dark-red",
-                  "index": 8,
-                  "text": "L6RSP"
-                },
-                "9": {
-                  "color": "dark-green",
-                  "index": 9,
-                  "text": "L7OK"
-                },
-                "10": {
-                  "color": "semi-dark-red",
-                  "index": 10,
-                  "text": "L7OKC"
-                },
-                "11": {
-                  "color": "dark-red",
-                  "index": 11,
-                  "text": "L7TOUT"
-                },
-                "12": {
-                  "color": "dark-red",
-                  "index": 12,
-                  "text": "L7RSP"
-                },
-                "13": {
-                  "color": "dark-red",
-                  "index": 13,
-                  "text": "L7STS"
-                }
-              },
-              "type": "value"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "id": 82,
-      "options": {
-        "alignValue": "left",
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom"
-        },
-        "mergeValues": true,
-        "rowHeight": 0.9,
-        "showValue": "auto",
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "redis_state",
-          "interval": "",
-          "legendFormat": "{{ servername }}",
-          "refId": "A"
-        }
-      ],
-      "title": "Redis State",
-      "type": "state-timeline"
-    },
-    {
-      "datasource": "Prometheus",
-      "description": "",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "fillOpacity": 70,
-            "lineWidth": 0
-          },
-          "mappings": [
-            {
-              "options": {
-                "0": {
-                  "color": "super-light-red",
-                  "index": 0,
-                  "text": "UNK"
-                },
-                "1": {
-                  "color": "semi-dark-yellow",
-                  "index": 1,
-                  "text": "INI"
-                },
-                "2": {
-                  "color": "semi-dark-red",
-                  "index": 2,
-                  "text": "SOCKERR"
-                },
-                "3": {
-                  "color": "dark-green",
-                  "index": 3,
-                  "text": "L4OK"
-                },
-                "4": {
-                  "color": "dark-red",
-                  "index": 4,
-                  "text": "L4TOUT"
-                },
-                "5": {
-                  "color": "dark-red",
-                  "index": 5,
-                  "text": "L4CON"
-                },
-                "6": {
-                  "color": "dark-green",
-                  "index": 6,
-                  "text": "L6OK"
-                },
-                "7": {
-                  "color": "dark-red",
-                  "index": 7,
-                  "text": "L6TOUT"
-                },
-                "8": {
-                  "color": "dark-red",
-                  "index": 8,
-                  "text": "L6RSP"
-                },
-                "9": {
-                  "color": "dark-green",
-                  "index": 9,
-                  "text": "L7OK"
-                },
-                "10": {
-                  "color": "semi-dark-red",
-                  "index": 10,
-                  "text": "L7OKC"
-                },
-                "11": {
-                  "color": "dark-red",
-                  "index": 11,
-                  "text": "L7TOUT"
-                },
-                "12": {
-                  "color": "dark-red",
-                  "index": 12,
-                  "text": "L7RSP"
-                },
-                "13": {
-                  "color": "dark-red",
-                  "index": 13,
-                  "text": "L7STS"
-                }
-              },
-              "type": "value"
-            }
-          ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "id": 83,
-      "options": {
-        "alignValue": "left",
-        "legend": {
-          "displayMode": "table",
-          "placement": "bottom"
-        },
-        "mergeValues": true,
-        "rowHeight": 0.9,
-        "showValue": "auto",
-        "tooltip": {
-          "mode": "single"
-        }
-      },
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "mongo_state",
-          "interval": "",
-          "legendFormat": "{{ servername }}",
-          "refId": "A"
-        }
-      ],
-      "title": "Mongo State",
-      "type": "state-timeline"
-    },
     {
       "collapsed": false,
       "datasource": null,
@@ -315,7 +50,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 8
+        "y": 0
       },
       "id": 46,
       "panels": [],
@@ -337,10 +72,10 @@
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
-        "h": 6,
+        "h": 8,
         "w": 8,
         "x": 0,
-        "y": 9
+        "y": 1
       },
       "hiddenSeries": false,
       "id": 22,
@@ -450,10 +185,10 @@
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
-        "h": 6,
+        "h": 8,
         "w": 8,
         "x": 8,
-        "y": 9
+        "y": 1
       },
       "hiddenSeries": false,
       "id": 44,
@@ -576,10 +311,10 @@
       "fill": 1,
       "fillGradient": 10,
       "gridPos": {
-        "h": 6,
+        "h": 8,
         "w": 8,
         "x": 16,
-        "y": 9
+        "y": 1
       },
       "hiddenSeries": false,
       "id": 38,
@@ -591,6 +326,7 @@
         "hideZero": false,
         "max": false,
         "min": false,
+        "rightSide": false,
         "show": true,
         "total": false,
         "values": true
@@ -637,7 +373,8 @@
             "include": {
               "names": [
                 "Time",
-                "feed_loadtimes_avg_hour"
+                "feed_loadtimes_avg_hour",
+                "feed_loadtimes_1min"
               ]
             }
           }
@@ -684,7 +421,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 15
+        "y": 9
       },
       "id": 20,
       "panels": [],
@@ -711,7 +448,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 16
+        "y": 10
       },
       "hiddenSeries": false,
       "id": 2,
@@ -817,7 +554,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 16
+        "y": 10
       },
       "hiddenSeries": false,
       "id": 6,
@@ -906,6 +643,138 @@
         "alignLevel": null
       }
     },
+    {
+      "datasource": "Prometheus",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "lineWidth": 0
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "super-light-red",
+                  "index": 0,
+                  "text": "UNK"
+                },
+                "1": {
+                  "color": "semi-dark-yellow",
+                  "index": 1,
+                  "text": "INI"
+                },
+                "2": {
+                  "color": "semi-dark-red",
+                  "index": 2,
+                  "text": "SOCKERR"
+                },
+                "3": {
+                  "color": "dark-green",
+                  "index": 3,
+                  "text": "L4OK"
+                },
+                "4": {
+                  "color": "dark-red",
+                  "index": 4,
+                  "text": "L4TOUT"
+                },
+                "5": {
+                  "color": "dark-red",
+                  "index": 5,
+                  "text": "L4CON"
+                },
+                "6": {
+                  "color": "dark-green",
+                  "index": 6,
+                  "text": "L6OK"
+                },
+                "7": {
+                  "color": "dark-red",
+                  "index": 7,
+                  "text": "L6TOUT"
+                },
+                "8": {
+                  "color": "dark-red",
+                  "index": 8,
+                  "text": "L6RSP"
+                },
+                "9": {
+                  "color": "dark-green",
+                  "index": 9,
+                  "text": "L7OK"
+                },
+                "10": {
+                  "color": "semi-dark-red",
+                  "index": 10,
+                  "text": "L7OKC"
+                },
+                "11": {
+                  "color": "dark-red",
+                  "index": 11,
+                  "text": "L7TOUT"
+                },
+                "12": {
+                  "color": "dark-red",
+                  "index": 12,
+                  "text": "L7RSP"
+                },
+                "13": {
+                  "color": "dark-red",
+                  "index": 13,
+                  "text": "L7STS"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 26,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 84,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "hidden",
+          "placement": "bottom"
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "never",
+        "tooltip": {
+          "mode": "single"
+        }
+      },
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "haproxy_state",
+          "interval": "",
+          "legendFormat": "{{ servername }}",
+          "refId": "A"
+        }
+      ],
+      "title": "Server health states via haproxy",
+      "type": "state-timeline"
+    },
     {
       "collapsed": false,
       "datasource": null,
@@ -913,7 +782,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 24
+        "y": 44
       },
       "id": 4,
       "panels": [],
@@ -950,7 +819,7 @@
         "h": 21,
         "w": 24,
         "x": 0,
-        "y": 25
+        "y": 45
       },
       "id": 10,
       "options": {
@@ -1004,7 +873,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 46
+        "y": 66
       },
       "hiddenSeries": false,
       "id": 48,
@@ -1095,7 +964,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 53
+        "y": 73
       },
       "id": 42,
       "panels": [],
@@ -1114,7 +983,7 @@
         "h": 9,
         "w": 24,
         "x": 0,
-        "y": 54
+        "y": 74
       },
       "hiddenSeries": false,
       "id": 33,
@@ -1217,7 +1086,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 63
+        "y": 83
       },
       "hiddenSeries": false,
       "id": 30,
@@ -1412,7 +1281,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 71
+        "y": 91
       },
       "hiddenSeries": false,
       "id": 32,
@@ -1511,7 +1380,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 79
+        "y": 99
       },
       "hiddenSeries": false,
       "id": 28,
@@ -1624,7 +1493,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 86
+        "y": 106
       },
       "hiddenSeries": false,
       "id": 12,
@@ -1718,7 +1587,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 86
+        "y": 106
       },
       "hiddenSeries": false,
       "id": 26,
@@ -1809,7 +1678,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 94
+        "y": 114
       },
       "hiddenSeries": false,
       "id": 14,
@@ -1895,7 +1764,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 102
+        "y": 122
       },
       "id": 50,
       "panels": [],
@@ -1914,7 +1783,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 103
+        "y": 123
       },
       "hiddenSeries": false,
       "id": 56,
@@ -2009,7 +1878,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 111
+        "y": 131
       },
       "hiddenSeries": false,
       "id": 52,
@@ -2102,7 +1971,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 111
+        "y": 131
       },
       "hiddenSeries": false,
       "id": 54,
@@ -2195,7 +2064,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 119
+        "y": 139
       },
       "hiddenSeries": false,
       "id": 58,
@@ -2288,7 +2157,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 119
+        "y": 139
       },
       "hiddenSeries": false,
       "id": 60,
@@ -2376,7 +2245,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 127
+        "y": 147
       },
       "id": 62,
       "panels": [],
@@ -2395,7 +2264,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 128
+        "y": 148
       },
       "hiddenSeries": false,
       "id": 75,
@@ -2439,7 +2308,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Redis DB Keus",
+      "title": "Redis DB Keys",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -2490,7 +2359,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 136
+        "y": 156
       },
       "hiddenSeries": false,
       "id": 66,
@@ -2651,7 +2520,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 144
+        "y": 164
       },
       "hiddenSeries": false,
       "id": 71,
@@ -2797,7 +2666,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 152
+        "y": 172
       },
       "hiddenSeries": false,
       "id": 72,
@@ -2929,7 +2798,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 160
+        "y": 180
       },
       "hiddenSeries": false,
       "id": 73,
@@ -3029,7 +2898,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 167
+        "y": 187
       },
       "hiddenSeries": false,
       "id": 78,
@@ -3122,7 +2991,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 175
+        "y": 195
       },
       "hiddenSeries": false,
       "id": 80,
@@ -3215,7 +3084,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 175
+        "y": 195
       },
       "hiddenSeries": false,
       "id": 64,
@@ -3303,7 +3172,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 183
+        "y": 203
       },
       "id": 68,
       "panels": [],
@@ -3322,7 +3191,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 184
+        "y": 204
       },
       "hiddenSeries": false,
       "id": 70,
@@ -3415,7 +3284,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 184
+        "y": 204
       },
       "hiddenSeries": false,
       "id": 76,
@@ -3503,7 +3372,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 192
+        "y": 212
       },
       "id": 16,
       "panels": [],
@@ -3522,7 +3391,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 193
+        "y": 213
       },
       "hiddenSeries": false,
       "id": 18,
@@ -3613,7 +3482,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 193
+        "y": 213
       },
       "hiddenSeries": false,
       "id": 24,
@@ -3704,7 +3573,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 201
+        "y": 221
       },
       "hiddenSeries": false,
       "id": 35,
@@ -3795,7 +3664,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 201
+        "y": 221
       },
       "hiddenSeries": false,
       "id": 8,
@@ -3881,7 +3750,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 209
+        "y": 229
       },
       "id": 40,
       "panels": [
@@ -3953,7 +3822,7 @@
       "type": "row"
     }
   ],
-  "refresh": "",
+  "refresh": "5s",
   "schemaVersion": 32,
   "style": "dark",
   "tags": [],
@@ -3968,5 +3837,5 @@
   "timezone": "",
   "title": "NewsBlur",
   "uid": "T86VjXrG2",
-  "version": 17
-}
+  "version": 62
+}

docker/grafana/dashboards/postgres_dashboard.json

@@ -1,486 +1,486 @@
 {
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "gnetId": null,
-  "graphTooltip": 0,
-  "id": 10,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 0
-      },
-      "hiddenSeries": false,
-      "id": 8,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_tup_updated{datname=\"newsblur\"}[$__interval])",
-          "interval": "",
-          "legendFormat": "rows updated",
-          "queryType": "randomWalk",
-          "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_tup_fetched{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "rows fetched",
-          "refId": "B"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_tup_inserted{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "rows inserted",
-          "refId": "C"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_tup_returned{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "rows returned",
-          "refId": "D"
-        },
-        {
-          "hide": false,
-          "refId": "E"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Row Counts",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:393",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "$$hashKey": "object:394",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+    "annotations": {
+        "list": [
+            {
+                "builtIn": 1,
+                "datasource": "-- Grafana --",
+                "enable": true,
+                "hide": true,
+                "iconColor": "rgba(0, 211, 255, 1)",
+                "name": "Annotations & Alerts",
+                "target": {
+                    "limit": 100,
+                    "matchAny": false,
+                    "tags": [],
+                    "type": "dashboard"
+                },
+                "type": "dashboard"
+            }
+        ]
     },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 0
-      },
-      "hiddenSeries": false,
-      "id": 6,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "gnetId": null,
+    "graphTooltip": 0,
+    "id": 10,
+    "links": [],
+    "liveNow": false,
+    "panels": [
         {
-          "exemplar": true,
-          "expr": "pg_settings_block_size",
-          "interval": "",
-          "legendFormat": "",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Block Size",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:329",
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 0
+            },
+            "hiddenSeries": false,
+            "id": 8,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_tup_updated{datname=\"newsblur\"}[$__interval])",
+                    "interval": "",
+                    "legendFormat": "rows updated",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_tup_fetched{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "rows fetched",
+                    "refId": "B"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_tup_inserted{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "rows inserted",
+                    "refId": "C"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_tup_returned{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "rows returned",
+                    "refId": "D"
+                },
+                {
+                    "hide": false,
+                    "refId": "E"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Row Counts",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:393",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": "0",
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:394",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
         },
         {
-          "$$hashKey": "object:330",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 12,
+                "y": 0
+            },
+            "hiddenSeries": false,
+            "id": 6,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "pg_settings_block_size",
+                    "interval": "",
+                    "legendFormat": "",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Block Size",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:329",
+                    "format": "bytes",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": "0",
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:330",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 8,
+                "w": 12,
+                "x": 0,
+                "y": 8
+            },
+            "hiddenSeries": false,
+            "id": 4,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts{datname=\"newsblur\"}[$__interval])",
+                    "interval": "",
+                    "legendFormat": "Database Conflicts",
+                    "queryType": "randomWalk",
+                    "refId": "All"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts_confl_bufferpin{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "Canceled due to pinned buffers",
+                    "refId": "Buffered Pin"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts_confl_deadlock{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "Canceled due to deadlocks",
+                    "refId": "Deadlocks"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts_confl_lock{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "Canceled due to lock timeouts",
+                    "refId": "Lock Timeouts"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts_confl_tablespace{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "Canceled due to dropped tablespaces",
+                    "refId": "A"
+                },
+                {
+                    "exemplar": true,
+                    "expr": "rate(pg_stat_database_conflicts_confl_snapshot{datname=\"newsblur\"}[$__interval])",
+                    "hide": false,
+                    "interval": "",
+                    "legendFormat": "Cancelled due to old snapshots",
+                    "refId": "B"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Conflicts",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:121",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:122",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
+        },
+        {
+            "aliasColors": {},
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "Prometheus",
+            "fill": 1,
+            "fillGradient": 0,
+            "gridPos": {
+                "h": 9,
+                "w": 12,
+                "x": 12,
+                "y": 8
+            },
+            "hiddenSeries": false,
+            "id": 2,
+            "legend": {
+                "avg": false,
+                "current": false,
+                "max": false,
+                "min": false,
+                "show": true,
+                "total": false,
+                "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "nullPointMode": "null",
+            "options": {
+                "alertThreshold": true
+            },
+            "percentage": false,
+            "pluginVersion": "8.2.6",
+            "pointradius": 2,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [],
+            "spaceLength": 10,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+                {
+                    "exemplar": true,
+                    "expr": "process_resident_memory_bytes{job=\"postgres exporter\"}",
+                    "interval": "",
+                    "legendFormat": "",
+                    "queryType": "randomWalk",
+                    "refId": "A"
+                }
+            ],
+            "thresholds": [],
+            "timeFrom": null,
+            "timeRegions": [],
+            "timeShift": null,
+            "title": "Memory",
+            "tooltip": {
+                "shared": true,
+                "sort": 0,
+                "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+                "buckets": null,
+                "mode": "time",
+                "name": null,
+                "show": true,
+                "values": []
+            },
+            "yaxes": [
+                {
+                    "$$hashKey": "object:51",
+                    "format": "bytes",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": "0",
+                    "show": true
+                },
+                {
+                    "$$hashKey": "object:52",
+                    "format": "short",
+                    "label": null,
+                    "logBase": 1,
+                    "max": null,
+                    "min": null,
+                    "show": true
+                }
+            ],
+            "yaxis": {
+                "align": false,
+                "alignLevel": null
+            }
         }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+    ],
+    "refresh": false,
+    "schemaVersion": 32,
+    "style": "dark",
+    "tags": [],
+    "templating": {
+        "list": []
     },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 8
-      },
-      "hiddenSeries": false,
-      "id": 4,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts{datname=\"newsblur\"}[$__interval])",
-          "interval": "",
-          "legendFormat": "Database Conflicts",
-          "queryType": "randomWalk",
-          "refId": "All"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts_confl_bufferpin{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Canceled due to pinned buffers",
-          "refId": "Buffered Pin"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts_confl_deadlock{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Canceled due to deadlocks",
-          "refId": "Deadlocks"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts_confl_lock{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Canceled due to lock timeouts",
-          "refId": "Lock Timeouts"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts_confl_tablespace{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Canceled due to dropped tablespaces",
-          "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "rate(pg_stat_database_conflicts_confl_snapshot{datname=\"newsblur\"}[$__interval])",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Cancelled due to old snapshots",
-          "refId": "B"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Conflicts",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:121",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:122",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+    "time": {
+        "from": "2021-10-26T11:57:59.481Z",
+        "to": "2021-10-26T14:21:53.820Z"
     },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 8
-      },
-      "hiddenSeries": false,
-      "id": 2,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.6",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "process_resident_memory_bytes{job=\"postgres exporter\"}",
-          "interval": "",
-          "legendFormat": "",
-          "queryType": "randomWalk",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Memory",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:51",
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "$$hashKey": "object:52",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    }
-  ],
-  "refresh": false,
-  "schemaVersion": 32,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": []
-  },
-  "time": {
-    "from": "2021-10-26T11:57:59.481Z",
-    "to": "2021-10-26T14:21:53.820Z"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "Postgres",
-  "uid": "0xQC_ednz",
-  "version": 1
+    "timepicker": {},
+    "timezone": "",
+    "title": "Postgres",
+    "uid": "0xQC_ednz",
+    "version": 2
 }

docker/haproxy/haproxy.consul.cfg.j2

@@ -66,7 +66,7 @@ frontend public
     use_backend node_images if { hdr_end(host) -i imageproxy2.newsblur.com }
     use_backend node_page if { path_beg /original_page/ }
     use_backend blog if { hdr_end(host) -i blog.newsblur.com }
-    use_backend blog if { hdr_end(host) -i blog2.newsblur.com }
+    use_backend sentry if { hdr_end(host) -i sentry.newsblur.com }
     use_backend nginx if { path_beg /media/ }
     use_backend nginx if { path_beg /static/ }
     use_backend nginx if { path_beg /favicon }
@@ -87,7 +87,7 @@ backend nginx
     http-check expect rstatus 200|503
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
     {% for host in groups.web %}
-        server {{host}}-nginx {{host}}.node.nyc1.consul:80
+        server nginx-{{host}} {{host}}.node.nyc1.consul:80
     {% endfor %}
 
 backend app_django
@@ -132,6 +132,14 @@ backend blog
         server {{host}} {{host}}.node.nyc1.consul:80
     {% endfor %}
 
+backend sentry
+    balance roundrobin
+    option httpchk GET /_health
+    default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    {% for host in groups.sentry %}
+        server {{host}} {{host}}.node.nyc1.consul:9000
+    {% endfor %}
+
 backend node_images
     option httpchk HEAD /sc,sN1megONJiGNy-CCvqzVPTv-TWRhgSKhFlf61XAYESl4=/http:/samuelclay.com/static/images/2019%20-%20Cuba.jpg
     http-check expect rstatus 200|301
@@ -198,25 +206,25 @@ backend db_redis_pubsub
 
 backend db_elasticsearch
     option httpchk GET /db_check/elasticsearch
-    server elasticsearch db-elasticsearch.node.nyc1.consul:5579 check inter 2000ms resolvers consul resolve-opts allow-dup-ip init-addr none
+    server db-elasticsearch db-elasticsearch.node.nyc1.consul:5579 check inter 2000ms resolvers consul resolve-opts allow-dup-ip init-addr none
 
 backend db_metrics
     balance roundrobin
     # option httpchk GET /_haproxychk
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
-    server grafana grafana.service.nyc1.consul:3000
+    server db-grafana grafana.service.nyc1.consul:3000
 
 backend consul_manager
     balance roundrobin
     # option httpchk GET /_haproxychk
     default-server check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
-    server consul_manager consul-manager.service.nyc1.consul:8500
+    server db-consul-manager consul-manager.service.nyc1.consul:8500
 
 backend maintenance
     option httpchk HEAD /maintenance
     http-check expect status 404
     http-check send-state
-    server nginx app-django1.node.nyc1.consul:80 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
+    server maintenance app-django1.node.nyc1.consul:80 check inter 2000ms resolvers consul resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
 
 listen stats
 bind :1936 ssl crt {{ ssl_certificate }}

docker/monitor/Dockerfile

@@ -11,7 +11,6 @@ RUN       set -ex \
           && buildDeps=' \
                     patch \
                     gfortran \
-                    lib32ncurses5-dev  \
                     libblas-dev \
                     libffi-dev \
                     libjpeg-dev \

docker/newsblur_base_image.Dockerfile

@@ -10,7 +10,6 @@ RUN       set -ex \
           && buildDeps=' \
                     patch \
                     gfortran \
-                    lib32ncurses5-dev  \
                     libblas-dev \
                     libffi-dev \
                     libjpeg-dev \

docker/newsblur_deploy.Dockerfile

@@ -0,0 +1,27 @@
+FROM newsblur/newsblur_python3
+ENV DOCKERBUILD=True
+
+RUN apt update
+RUN apt install -y curl
+
+# Install Java
+# Install OpenJDK-11
+RUN apt install -y openjdk-11-jre-headless
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/
+RUN export JAVA_HOME
+WORKDIR /tmp
+RUN apt install wget unzip
+RUN wget "https://dl.google.com/closure-compiler/compiler-20200719.zip"
+RUN unzip "compiler-20200719.zip"
+RUN mv closure-compiler-v20200719.jar /usr/local/bin/compiler.jar
+
+# Install Node
+RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash -
+RUN apt install -y nodejs build-essential
+RUN	npm -g install yuglify
+
+# Cleanup
+RUN apt-get clean
+
+WORKDIR /srv/newsblur
+CMD python manage.py collectstatic --no-input --clear -v 1 -l

docker/nginx/Dockerfile

@@ -1,2 +0,0 @@
-FROM nginx:latest
-COPY ./docker/nginx /etc/nginx/conf.d

docker/nginx/nginx.consul.conf.j2

@@ -61,6 +61,7 @@ server {
     }
 
     location /static/  {
+        gzip_static on;
         expires max;
         keepalive_timeout 1;
         root /srv/newsblur;

docker/nginx/nginx.local.conf

@@ -64,6 +64,7 @@ server {
     }
 
     location /static/  {
+        gzip_static on;
         expires max;
         keepalive_timeout 1;
         root /srv/newsblur;

docker/postgres/postgresql-13.conf.j2

@@ -38,7 +38,7 @@
 # The default values of these variables are driven from the -D command-line
 # option or PGDATA environment variable, represented here as ConfigDir.
 
-data_directory = '/var/lib/postgresql/13/pgdata'		# use data in another directory
+data_directory = '/var/lib/postgresql/data'		# use data in another directory
 					# (change requires restart)
 hba_file = '/etc/postgresql/pg_hba.conf'	# host-based authentication file
 					# (change requires restart)
@@ -235,7 +235,7 @@ min_wal_size = 80MB
 
 archive_mode = on		# enables archiving; off, on, or always
 				# (change requires restart)
-archive_command = 'test ! -f ../archive/%f && cp -f %p ../archive/%f'
+archive_command = 'test ! -f /var/lib/postgresql/archive/%f && cp -f %p /var/lib/postgresql/archive/%f'
 				# placeholders: %p = path of file to archive
 				#               %f = file name only
 				# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
@@ -583,7 +583,7 @@ track_counts = on
 #track_io_timing = off
 #track_functions = none			# none, pl, all
 #track_activity_query_size = 1024	# (change requires restart)
-stats_temp_directory = '/var/run/postgresql/13-main.pg_stat_tmp'
+stats_temp_directory = '/var/run/postgresql/'
 
 
 # - Monitoring -

docker/prometheus/prometheus.consul.yml.j2

@@ -238,19 +238,12 @@ scrape_configs:
       - source_labels: ['__meta_consul_node']
         target_label: instance
 
-    - job_name: 'redis state'
+    - job_name: 'haproxy state'
       consul_sd_configs:
       - server: 'consul.service.nyc1.consul:8500'
-        services: ['flask_metrics_redis']
-      relabel_configs:
-      - source_labels: ['__meta_consul_node']
-        target_label: instance
-      metrics_path: /state/
-    - job_name: 'mongo state'
-      consul_sd_configs:
-      - server: 'consul.service.nyc1.consul:8500'
-        services: ['flask_metrics_mongo']
+        services: ['flask_metrics_haproxy']
       relabel_configs:
       - source_labels: ['__meta_consul_node']
         target_label: instance
       metrics_path: /state/
+      scrape_interval: 5s

docker/prometheus/prometheus.docker.yml

@@ -175,42 +175,42 @@ scrape_configs:
 
     - job_name: 'redis active connections'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /active-connections/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis commands'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /commands/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis connects'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /connects/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis size'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /size/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis memory'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /memory/
       scheme: http
       tls_config:
         insecure_skip_verify: true
     - job_name: 'redis used memory'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_redis:5589']
       metrics_path: /used-memory/
       scheme: http
       tls_config:
@@ -230,17 +230,11 @@ scrape_configs:
       tls_config:
         insecure_skip_verify: true
 
-    - job_name: 'redis state'
+    - job_name: 'haproxy state'
       static_configs:
-        - targets: ['flask_metrics_redis:5569']
+        - targets: ['flask_metrics_haproxy:5599']
       metrics_path: /state/
       scheme: http
       tls_config:
         insecure_skip_verify: true
-    - job_name: 'mongo state'
-      static_configs:
-        - targets: ['flask_metrics_mongo:5569']
-      metrics_path: /state/
-      scheme: http
-      tls_config:
-        insecure_skip_verify: true
+      scrape_interval: 30s

flask_metrics/flask_metrics_haproxy.py

@@ -0,0 +1,81 @@
+from flask import Flask, render_template, Response
+from newsblur_web import settings
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+import requests
+from requests.auth import HTTPBasicAuth
+
+if settings.FLASK_SENTRY_DSN is not None:
+    sentry_sdk.init(
+        dsn=settings.FLASK_SENTRY_DSN,
+        integrations=[FlaskIntegration()],
+        traces_sample_rate=1.0,
+    )
+
+app = Flask(__name__)
+
+if settings.DOCKERBUILD:
+    pass
+
+
+STATUS_MAPPING = {
+    "UNK": 0,      # unknown
+    "INI": 1,      # initializing
+    "SOCKERR": 2,  # socket error
+    "L4OK": 3,     # check passed on layer 4, no upper layers testing enabled
+    "L4TOUT": 4,   # layer 1-4 timeout
+    "L4CON": 5,    # layer 1-4 connection problem, for example "Connection refused" (tcp rst) or "No route to host" (icmp)
+    "L6OK": 6,     # check passed on layer 6
+    "L6TOUT": 7,   # layer 6 (SSL) timeout
+    "L6RSP": 8,    # layer 6 invalid response - protocol error
+    "L7OK": 9,     # check passed on layer 7
+    "L7OKC": 10,   # check conditionally passed on layer 7, for example 404 with disable-on-404
+    "L7TOUT": 11,  # layer 7 (HTTP/SMTP) timeout
+    "L7RSP": 12,   # layer 7 invalid response - protocol error
+    "L7STS": 13,   # layer 7 response error, for example HTTP 5xx
+}
+
+def format_state_data(label, data):
+    formatted_data = {}
+    for k, v in data.items():
+        if v:
+            formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
+    return formatted_data
+
+def fetch_states():
+    res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
+
+    lines = res.content.decode('utf-8').split('\n')
+    header_line = lines[0].split(",")
+    check_status_index = header_line.index('check_status')
+    servername_index = header_line.index('svname')
+
+    data = {}
+    backends = [line.split(",") for line in lines[1:]]
+    for backend_data in backends:
+        if len(backend_data) <= check_status_index: continue
+        if len(backend_data) <= servername_index: continue
+        if backend_data[servername_index] in ['FRONTEND', 'BACKEND']: continue
+        backend_status = backend_data[check_status_index].replace("*", "")
+        data[backend_data[servername_index]] = backend_status
+    
+    return data
+
+
+@app.route("/state/")
+def haproxy_state():
+    backends = fetch_states()
+    
+    formatted_data = format_state_data("haproxy_state", backends)
+    context = {
+        'chart_name': 'haproxy_state',
+        'chart_type': 'gauge',
+        'data': formatted_data
+    }
+    html_body = render_template('prometheus_data.html', **context)
+    return Response(html_body, content_type="text/plain")
+
+
+if __name__ == "__main__":
+    print(" ---> Starting NewsBlur Flask Metrics server for HAProxy...")
+    app.run(host="0.0.0.0", port=5569, debug=settings.DEBUG)

flask_metrics/flask_metrics_mongo.py

@@ -1,6 +1,5 @@
 from flask import Flask, render_template, Response
 import pymongo
-from flask_metrics.state_timeline import format_state_data, get_state
 from newsblur_web import settings
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
@@ -166,20 +165,6 @@ def page_queues():
     html_body = render_template('prometheus_data.html', **context)
     return Response(html_body, content_type="text/plain")
 
-@app.route("/state/")
-def mongo_state():
-    mongo_data = get_state("mongo")
-    if 'BACKEND' in mongo_data:
-        del mongo_data['BACKEND']
-    formatted_data = format_state_data("mongo_state", mongo_data)
-    context = {
-        'chart_name': 'mongo_state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
 
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")

flask_metrics/flask_metrics_redis.py

@@ -1,6 +1,5 @@
 from flask import Flask, render_template, Response
 from newsblur_web import settings
-from flask_metrics.state_timeline import format_state_data, get_state
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
 import redis
@@ -187,19 +186,6 @@ def memory_used():
     return Response(html_body, content_type="text/plain")
 
 
-@app.route("/state/")
-def redis_state():
-    redis_data = get_state("db_redis")
-    formatted_data = format_state_data("redis_state", redis_data)
-    context = {
-        'chart_name': 'redis_state',
-        'chart_type': 'gauge',
-        'data': formatted_data
-    }
-    html_body = render_template('prometheus_data.html', **context)
-    return Response(html_body, content_type="text/plain")
-
-
 if __name__ == "__main__":
     print(" ---> Starting NewsBlur Flask Metrics server...")
     app.run(host="0.0.0.0", port=5569)

flask_metrics/state_timeline.py

@@ -1,42 +0,0 @@
-from flask import render_template
-import requests
-from requests.auth import HTTPBasicAuth
-
-
-STATUS_MAPPING = {
-    "UNK": 0,      # unknown
-    "INI": 1,      # initializing
-    "SOCKERR": 2,  # socket error
-    "L4OK": 3,     # check passed on layer 4, no upper layers testing enabled
-    "L4TOUT": 4,   # layer 1-4 timeout
-    "L4CON": 5,    # layer 1-4 connection problem, for example "Connection refused" (tcp rst) or "No route to host" (icmp)
-    "L6OK": 6,     # check passed on layer 6
-    "L6TOUT": 7,   # layer 6 (SSL) timeout
-    "L6RSP": 8,    # layer 6 invalid response - protocol error
-    "L7OK": 9,     # check passed on layer 7
-    "L7OKC": 10,   # check conditionally passed on layer 7, for example 404 with disable-on-404
-    "L7TOUT": 11,  # layer 7 (HTTP/SMTP) timeout
-    "L7RSP": 12,   # layer 7 invalid response - protocol error
-    "L7STS": 13,   # layer 7 response error, for example HTTP 5xx
-}
-
-def format_state_data(label, data):
-    formatted_data = {}
-    for k, v in data.items():
-        if v:
-            formatted_data[k] = f'{label}{{servername="{k}"}} {STATUS_MAPPING[v.strip()]}'
-    return formatted_data
-
-def get_state(backend_name):
-    res = requests.get('https://newsblur.com:1936/;csv', auth=HTTPBasicAuth('gimmiestats', 'StatsGiver'))
-    lines = res.content.decode('utf-8').split('\n')
-    backends = [line.split(",") for line in lines if backend_name in line]
-
-    check_status_index = lines[0].split(",").index('check_status')
-    servername_index = lines[0].split(",").index('svname')
-
-    data = {}
-
-    for backend_data in backends:
-        data[backend_data[servername_index]] = backend_data[check_status_index].replace("*", "")
-    return data

flask_monitor/db_monitor.py

@@ -15,7 +15,7 @@ from sentry_sdk.integrations.flask import FlaskIntegration
 sentry_sdk.init(
     dsn=settings.FLASK_SENTRY_DSN,
     integrations=[FlaskIntegration()],
-    traces_sample_rate=1.0,
+    traces_sample_rate=0.001,
 )
 
 app = Flask(__name__)

media/admin

@@ -1 +0,0 @@
-/usr/local/lib/python3.9/site-packages/django/contrib/admin/static/admin

media/android

@@ -1 +0,0 @@
-../clients/android