merge master

Makefile

@@ -153,6 +153,9 @@ firewall:
 	- ansible-playbook ansible/all.yml -l db --tags firewall
 oldfirewall:
 	- ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml  -l db --tags firewall
+repairmongo:
+	- sudo docker run -v "/srv/newsblur/docker/volumes/db_mongo:/data/db" mongo:4.0 mongod --repair --dbpath /data/db
+
 
 # performance tests
 perf-cli:

README.md

@@ -59,6 +59,12 @@
 
     Note: You will be warned that you are using a self signed certificate. In order to get around this warning you must type "thisisunsafe" as per https://dblazeski.medium.com/chrome-bypass-net-err-cert-invalid-for-development-daefae43eb12
 
+  3. To change the domain from localhost, you'll need to change it in a few places:
+
+   * Change `NEWSBLUR_URL` and `SESSION_COOKIE_DOMAIN` in `newsblur_web/docker_local_settings.py`
+   * Change the domain in `config/fixtures/bootstrap.json`, or if you've already created a site, edit the `Site.objects.all()[0]` domain in the shell, which you can access with `make shell`
+   * If you're using a custom subdomain, you'll also want to add it to `ALLOWED_SUBDOMAINS` in `apps/reader/views.py`
+
 ## Making docker-compose work with your database
 
 To make docker-compose work with your database, upgrade your local database to the docker-compose version and then volumize the database data path by changing the `./docker/volumes/` part of the volume directive in the service to point to your local database's data directory.

ansible/roles/elasticsearch/tasks/main.yml

@@ -15,6 +15,13 @@
     owner: nb
     group: nb
 
+- name: System max_map_count increase
+  become: yes
+  ansible.posix.sysctl:
+    name: vm.max_map_count
+    value: "262144"
+    state: present
+    
 - name: Make docker network for newsblurnet
   become: yes
   docker_network:
@@ -30,6 +37,7 @@
     hostname: "{{ inventory_hostname }}"
     ports:
       - '9200:9200'
+      - '9300:9300'
     restart_policy: unless-stopped
     container_default_behavior: no_defaults
     networks_cli_compatible: yes
@@ -43,6 +51,7 @@
     volumes:
       - /srv/newsblur/docker/volumes/elasticsearch:/usr/share/elasticsearch/data
       - /var/log/elasticsearch/:/var/log/elasticsearch/
+      - /srv/newsblur/config/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
 
 - name: Register elasticsearch in consul
   tags: consul

ansible/roles/haproxy/tasks/main.yml

@@ -89,9 +89,8 @@
     container_default_behavior: no_defaults
     command: "haproxy -f /srv/newsblur/docker/haproxy/haproxy.consul.cfg"
     volumes:
-      - /srv/newsblur:/srv/newsblur:consistent
-      - /etc/letsencrypt/live/newsblur.com:/etc/letsencrypt/live/newsblur.com:consistent
-      - /etc/letsencrypt/archive/newsblur.com:/etc/letsencrypt/archive/newsblur.com:consistent
+      - /srv/newsblur:/srv/newsblur
+      - /etc/letsencrypt:/etc/letsencrypt
 
 - name: Reload haproxy
   debug: 

ansible/roles/monitor/tasks/main.yml

@@ -30,6 +30,7 @@
     pull: yes
     state: started
     command: bash -c "python /srv/newsblur/flask_monitor/db_monitor.py"
+    hostname: "{{ inventory_hostname }}"
     restart_policy: unless-stopped
     container_default_behavior: no_defaults
     volumes:

ansible/roles/web/files/logrotate.conf

@@ -7,7 +7,6 @@
        compress
        notifempty
        missingok
-       su nb nb
 }
 
 /var/log/nginx/*.log {

ansible/roles/web/tasks/main.yml

@@ -85,6 +85,8 @@
 - name: Setup logrotate
   become: yes
   copy: src=logrotate.conf dest=/etc/logrotate.d/newsblur mode=0755
+  tags:
+    - logrotate
 
 - name: Reload gunicorn
   debug: 

apps/reader/views.py

@@ -1126,16 +1126,18 @@ def starred_stories_rss_feed_tag(request, user_id, secret_token, tag_slug):
             user_id=user.pk,
             user_tags__contains=tag_counts.tag
         ).order_by('-starred_date').limit(25)
+
+    starred_stories = Feed.format_stories(starred_stories)
+
     for starred_story in starred_stories:
         story_data = {
-            'title': starred_story.story_title,
-            'link': starred_story.story_permalink,
-            'description': (starred_story.story_content_z and
-                            zlib.decompress(starred_story.story_content_z)),
-            'author_name': starred_story.story_author_name,
-            'categories': starred_story.story_tags,
-            'unique_id': starred_story.story_guid,
-            'pubdate': starred_story.starred_date,
+            'title': smart_str(starred_story['story_title']),
+            'link': starred_story['story_permalink'],
+            'description': smart_str(starred_story['story_content']),
+            'author_name': starred_story['story_authors'],
+            'categories': starred_story['story_tags'],
+            'unique_id': starred_story['story_permalink'],
+            'pubdate': starred_story['starred_date'],
         }
         rss.add_item(**story_data)
         

apps/rss_feeds/icon_importer.py

@@ -109,11 +109,10 @@ class IconImporter(object):
         base64.b64decode(image_str)
         settings.S3_CONN.Object(settings.S3_ICONS_BUCKET_NAME, 
                                 self.feed.s3_icons_key).put(Body=base64.b64decode(image_str), 
-                                                            ExtraArgs={
-                                                                'Content-Type': 'image/png',
-                                                                'Expires': expires,
-                                                                'ACL': 'public-read',
-                                                            })
+                                                            ContentType='image/png',
+                                                            Expires=expires,
+                                                            ACL='public-read'
+                                                            )
 
         self.feed.s3_icon = True
         self.feed.save()

apps/rss_feeds/models.py

@@ -2200,6 +2200,10 @@ class Feed(models.Model):
         # Craigslist feeds get 6 hours minimum
         if 'craigslist' in self.feed_address:
             total = max(total, 60*6)
+
+        # Twitter feeds get 2 hours minimum
+        if 'twitter' in self.feed_address:
+            total = max(total, 60*2)
                 
         if verbose:
             logging.debug("   ---> [%-30s] Fetched every %s min - Subs: %s/%s/%s Stories/day: %s" % (

apps/rss_feeds/page_importer.py

@@ -325,13 +325,11 @@ class PageImporter(object):
         s3_object = settings.S3_CONN.Object(settings.S3_PAGES_BUCKET_NAME, 
                                             self.feed.s3_pages_key)
         s3_object.put(Body=compress_string_with_gzip(html.encode('utf-8')), 
-                      ExtraArgs={
-                          'Content-Type': 'text/html',
-                          'Content-Encoding': 'gzip',
-                          'Access-Control-Allow-Origin': '*',
-                          'Expires': expires,
-                          'ACL': 'public-read',
-                      })
+                      ContentType='text/html',
+                      ContentEncoding='gzip',
+                      Expires=expires,
+                      ACL='public-read'
+                      )
         
         try:
             feed_page = MFeedPage.objects.get(feed_id=self.feed.pk)

apps/search/models.py

@@ -207,7 +207,7 @@ class SearchStory:
     
     @classmethod
     def doc_type(cls):
-        if settings.DOCKERBUILD:
+        if settings.DOCKERBUILD or getattr(settings, 'ES_IGNORE_TYPE', False):
             return None
         return "%s-type" % cls.name
         
@@ -230,33 +230,39 @@ class SearchStory:
             logging.debug(" ***> ~FRCould not create search index for ~FM%s: %s" % (cls.index_name(), e))
             return
         except (elasticsearch.exceptions.ConnectionError, 
-                urllib3.exceptions.NewConnectionError, 
+                urllib3.exceptions.NewConnectionError,
                 urllib3.exceptions.ConnectTimeoutError) as e:
-            logging.debug(f" ***> ~FRNo search server available for creating story mapping: {e}")
+            logging.debug(
+                f" ***> ~FRNo search server available for creating story mapping: {e}")
             return
-        
-        mapping = { 
+
+        mapping = {
             'title': {
-                'boost': 3.0,
                 'store': False,
                 'type': 'text',
                 'analyzer': 'snowball',
+                "term_vector": "yes",
             },
             'content': {
-                'boost': 1.0,
                 'store': False,
                 'type': 'text',
                 'analyzer': 'snowball',
+                "term_vector": "yes",
             },
             'tags': {
-                'boost': 2.0,
                 'store': False,
-                'type': 'keyword',
+                "type": "text",
+                "fields": {
+                    "raw": {
+                        "type": "text",
+                        "analyzer": "keyword",
+                        "term_vector": "yes"
+                    }
+                }
             },
             'author': {
-                'boost': 1.0,
                 'store': False,
-                'type': 'text',   
+                'type': 'text',
                 'analyzer': 'simple',
             },
             'feed_id': {
@@ -274,34 +280,35 @@ class SearchStory:
         cls.ES().indices.flush(cls.index_name())
 
     @classmethod
-    def index(cls, story_hash, story_title, story_content, story_tags, story_author, story_feed_id, 
+    def index(cls, story_hash, story_title, story_content, story_tags, story_author, story_feed_id,
               story_date):
         cls.create_elasticsearch_mapping()
 
         doc = {
-            "content"   : story_content,
-            "title"     : story_title,
-            "tags"      : ', '.join(story_tags),
-            "author"    : story_author,
-            "feed_id"   : story_feed_id,
-            "date"      : story_date,
+            "content": story_content,
+            "title": story_title,
+            "tags": ', '.join(story_tags),
+            "author": story_author,
+            "feed_id": story_feed_id,
+            "date": story_date,
         }
         try:
-            cls.ES().create(index=cls.index_name(), id=story_hash, body=doc, doc_type=cls.doc_type())
-        except (elasticsearch.exceptions.ConnectionError, 
+            cls.ES().create(index=cls.index_name(), id=story_hash,
+                            body=doc, doc_type=cls.doc_type())
+        except (elasticsearch.exceptions.ConnectionError,
                 urllib3.exceptions.NewConnectionError) as e:
-            logging.debug(f" ***> ~FRNo search server available for story indexing: {e}")
+            logging.debug(
+                f" ***> ~FRNo search server available for story indexing: {e}")
         except elasticsearch.exceptions.ConflictError as e:
             logging.debug(f" ***> ~FBAlready indexed story: {e}")
         # if settings.DEBUG:
         #     logging.debug(f" ***> ~FBIndexed {story_hash}")
 
-    
     @classmethod
     def remove(cls, story_hash):
         if not cls.ES().exists(index=cls.index_name(), id=story_hash, doc_type=cls.doc_type()):
             return
-        
+
         try:
             cls.ES().delete(index=cls.index_name(), id=story_hash, doc_type=cls.doc_type())
         except elasticsearch.exceptions.NotFoundError:
@@ -443,7 +450,7 @@ class SearchFeed:
         
     @classmethod
     def doc_type(cls):
-        if settings.DOCKERBUILD:
+        if settings.DOCKERBUILD or getattr(settings, 'ES_IGNORE_TYPE', False):
             return None
         return "%s-type" % cls.name
         

apps/social/models.py

@@ -161,6 +161,9 @@ class MSocialProfile(mongo.Document):
             profile = cls.objects.create(user_id=user_id)
             profile.save()
 
+        if not profile.username:
+            profile.save()
+
         return profile
     
     @property
@@ -171,6 +174,8 @@ class MSocialProfile(mongo.Document):
             return None
 
     def save(self, *args, **kwargs):
+        if not self.username:
+            self.import_user_fields()
         if not self.subscription_count:
             self.count_follows(skip_save=True)
         if self.bio and len(self.bio) > MSocialProfile.bio.max_length:
@@ -433,6 +438,11 @@ class MSocialProfile(mongo.Document):
             return [u for u in self.follower_user_ids if u != self.user_id]
         return self.follower_user_ids
         
+    def import_user_fields(self):
+        user = User.objects.get(pk=self.user_id)
+        self.username = user.username
+        self.email = user.email
+        
     def count_follows(self, skip_save=False):
         self.subscription_count = UserSubscription.objects.filter(user__pk=self.user_id).count()
         self.shared_stories_count = MSharedStory.objects.filter(user_id=self.user_id).count()

config/elasticsearch/elasticsearch.yml

@@ -1,6 +1,8 @@
 http.cors.enabled: true
 http.cors.allow-origin: "*"
 
+discovery.type: single-node
+
 cluster.routing.allocation.disk.threshold_enabled: false
 
 cluster.name: "docker-cluster"

flask_monitor/db_monitor.py

@@ -209,11 +209,11 @@ def db_check_redis_pubsub():
 @app.route("/db_check/elasticsearch")
 def db_check_elasticsearch():
     try:
-        conn = elasticsearch.Elasticsearch('db-elasticsearch.service.nyc1.consul')
+        conn = elasticsearch.Elasticsearch("elasticsearch")
     except:
         abort(503)
     
-    if conn.indices.exists_index('feeds-index'):
+    if conn.indices.exists('feeds-index'):
         return str("Index exists, but didn't try search")
         # query = pyes.query.TermQuery("title", "daring fireball")
         # results = conn.search(query=query, size=1, doc_types=['feeds-type'], sort="num_subscribers:desc")

flask_monitor/requirements.txt

@@ -9,3 +9,4 @@ Django>=3.1,<3.2
 sentry-sdk[flask]
 mongoengine==0.21.0
 boto3==1.18.13
+pyyaml==5.3.1

media/admin

@@ -1 +1 @@
-../venv/newsblur/lib/python2.7/site-packages/django/contrib/admin/static/admin
+/usr/local/lib/python3.9/site-packages/django/contrib/admin/static/admin

utils/s3_utils.py

@@ -2,8 +2,6 @@ import os
 import sys
 import time
 import mimetypes
-#from boto.s3.connection import S3Connection
-#from boto.s3.key import Key
 from utils.image_functions import ImageOps
 
 if '/srv/newsblur' not in ' '.join(sys.path):
@@ -75,7 +73,7 @@ class S3Store:
 
         if content_type:
             s3_object.put(Body=file_object, ExtraArgs={
-                'Content-Type': content_type,
+                'ContentType': content_type,
                 'ACL': 'public-read',
             })
         else: