public-mirrors/NewsBlur-viq
1
0
Fork 0
mirror of https://github.com/viq/NewsBlur.git synced 2025-08-05 16:49:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Upgrading elasticsearch to 1.7.0, pyes to 0.99.5, and modern querying for stories. Needs a reindex. Still need to do feeds.

Browse source
This commit is contained in:
Samuel Clay 2015-07-17 14:01:38 -07:00
parent fe52578879
commit b51a054c1c
3 changed files with 17 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
apps/search/management/commands/index_stories.py
View file

@ -1,16 +1,26 @@
import re import re
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.contrib.auth.models import User from django.contrib.auth.models import User
from apps.rss_feeds.models import Feed from apps.rss_feeds.models import Feed, MStory
from apps.reader.models import UserSubscription from apps.reader.models import UserSubscription
from optparse import make_option from optparse import make_option
class Command(BaseCommand): class Command(BaseCommand):
option_list = BaseCommand.option_list + ( option_list = BaseCommand.option_list + (
make_option("-u", "--user", dest="user", nargs=1, help="Specify user id or username"), make_option("-u", "--user", dest="user", nargs=1, help="Specify user id or username"),
make_option("-R", "--reindex", dest="reindex", action="store_true", help="Drop index and reindex all stories."),
) )
def handle(self, *args, **options): def handle(self, *args, **options):
if options['reindex']:
MStory.index_all_for_search()
return
if not options['user']:
print "Missing user. Did you want to reindex everything? Use -R."
return
if re.match(r"([0-9]+)", options['user']): if re.match(r"([0-9]+)", options['user']):
user = User.objects.get(pk=int(options['user'])) user = User.objects.get(pk=int(options['user']))
else: else:

10
apps/search/models.py
View file

@ -202,28 +202,28 @@ class SearchStory:
'index': 'analyzed', 'index': 'analyzed',
'store': 'no', 'store': 'no',
'type': 'string', 'type': 'string',
'analyzer': 'snowball', 'analyzer': 'standard',
}, },
'content': { 'content': {
'boost': 1.0, 'boost': 1.0,
'index': 'analyzed', 'index': 'analyzed',
'store': 'no', 'store': 'no',
'type': 'string', 'type': 'string',
'analyzer': 'snowball', 'analyzer': 'simple',
}, },
'tags': { 'tags': {
'boost': 2.0, 'boost': 2.0,
'index': 'analyzed', 'index': 'analyzed',
'store': 'no', 'store': 'no',
'type': 'string', 'type': 'string',
'analyzer': 'snowball', 'analyzer': 'standard',
}, },
'author': { 'author': {
'boost': 1.0, 'boost': 1.0,
'index': 'analyzed', 'index': 'analyzed',
'store': 'no', 'store': 'no',
'type': 'string', 'type': 'string',
'analyzer': 'keyword', 'analyzer': 'simple',
}, },
'feed_id': { 'feed_id': {
'store': 'no', 'store': 'no',
@ -272,7 +272,7 @@ class SearchStory:
cls.ES.indices.refresh() cls.ES.indices.refresh()
sort = "date:desc" if order == "newest" else "date:asc" sort = "date:desc" if order == "newest" else "date:asc"
string_q = pyes.query.StringQuery(query, default_operator="AND") string_q = pyes.query.QueryStringQuery(query, default_operator="AND")
feed_q = pyes.query.TermsQuery('feed_id', feed_ids[:1000]) feed_q = pyes.query.TermsQuery('feed_id', feed_ids[:1000])
q = pyes.query.BoolQuery(must=[string_q, feed_q]) q = pyes.query.BoolQuery(must=[string_q, feed_q])
try: try:

2
config/requirements.txt
View file

@ -27,7 +27,7 @@ mongoengine==0.8.2
nltk==2.0.5 nltk==2.0.5
oauth2==1.5.211 oauth2==1.5.211
psutil==2.1.0 psutil==2.1.0
pyes==0.90.1 pyes==0.99.5
simplejson==3.4.0 simplejson==3.4.0
pyflakes==0.6.1 pyflakes==0.6.1
pymongo==2.6 pymongo==2.6