public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-11-01 09:09:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
ojiikun 2013-04-08 23:42:27 +00:00
commit 9707b2faf2
38 changed files with 1329 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/reader/models.py
View file

@ -667,6 +667,7 @@ class MUserStory(mongo.Document):
if self.story_db_id:
all_read_stories_key = 'RS:%s' % (self.user_id)
r.sadd(all_read_stories_key, self.story_db_id)
r.expire(all_read_stories_key, settings.DAYS_OF_UNREAD*24*60*60)
read_story_key = 'RS:%s:%s' % (self.user_id, self.feed_id)
r.sadd(read_story_key, self.story_db_id)

2
apps/rss_feeds/management/commands/refresh_feeds.py
View file

@ -32,8 +32,6 @@ class Command(BaseCommand):
if options['daemonize']:
daemonize()
options['fake'] = bool(MStatistics.get('fake_fetch'))
settings.LOG_TO_STREAM = True
now = datetime.datetime.utcnow()

79
apps/rss_feeds/models.py
View file

@ -8,6 +8,7 @@ import mongoengine as mongo
import zlib
import hashlib
import redis
from urlparse import urlparse
from utils.feed_functions import Counter
from collections import defaultdict
from operator import itemgetter
@ -35,7 +36,8 @@ from utils.feed_functions import levenshtein_distance
from utils.feed_functions import timelimit, TimeoutError
from utils.feed_functions import relative_timesince
from utils.feed_functions import seconds_timesince
from utils.story_functions import strip_tags, htmldiff, strip_comments
from utils.story_functions import strip_tags, htmldiff, strip_comments, strip_comments__lxml
from vendor.redis_completion.engine import RedisEngine
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
@ -191,7 +193,6 @@ class Feed(models.Model):
try:
super(Feed, self).save(*args, **kwargs)
return self
except IntegrityError, e:
logging.debug(" ---> ~FRFeed save collision (%s), checking dupe..." % e)
duplicate_feeds = Feed.objects.filter(feed_address=self.feed_address,
@ -209,8 +210,10 @@ class Feed(models.Model):
logging.debug(" ---> ~FRFound different feed (%s), merging..." % duplicate_feeds[0])
feed = Feed.get_by_id(merge_feeds(duplicate_feeds[0].pk, self.pk))
return feed
return self
self.sync_autocompletion()
return self
def index_for_search(self):
if self.num_subscribers > 1 and not self.branch_from_feed:
@ -223,6 +226,31 @@ class Feed(models.Model):
def sync_redis(self):
return MStory.sync_all_redis(self.pk)
def sync_autocompletion(self):
if self.num_subscribers <= 1: return
if self.branch_from_feed: return
if any(t in self.feed_address for t in ['token', 'private']): return
engine = RedisEngine(prefix="FT", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL)
engine.store(self.pk, title=self.feed_title)
engine.boost(self.pk, self.num_subscribers)
parts = urlparse(self.feed_address)
engine = RedisEngine(prefix="FA", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL)
engine.store(self.pk, title=parts.hostname)
engine.boost(self.pk, self.num_subscribers)
@classmethod
def autocomplete(self, prefix, limit=5):
engine = RedisEngine(prefix="FT", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL)
results = engine.search(phrase=prefix, limit=limit, autoboost=True)
if len(results) < limit:
engine = RedisEngine(prefix="FA", connection_pool=settings.REDIS_AUTOCOMPLETE_POOL)
results += engine.search(phrase=prefix, limit=limit-len(results), autoboost=True, filters=[lambda f: f not in results])
return results
@classmethod
def find_or_create(cls, feed_address, feed_link, *args, **kwargs):
@ -335,8 +363,9 @@ class Feed(models.Model):
p.zadd('tasked_feeds', feed_id, now)
p.execute()
for feed_ids in (feeds[pos:pos + queue_size] for pos in xrange(0, len(feeds), queue_size)):
UpdateFeeds.apply_async(args=(feed_ids,), queue='update_feeds')
# for feed_ids in (feeds[pos:pos + queue_size] for pos in xrange(0, len(feeds), queue_size)):
for feed_id in feeds:
UpdateFeeds.apply_async(args=(feed_id,), queue='update_feeds')
def update_all_statistics(self, full=True, force=False):
self.count_subscribers()
@ -773,14 +802,14 @@ class Feed(models.Model):
feed.last_update = datetime.datetime.utcnow()
feed.set_next_scheduled_update()
r.zadd('fetched_feeds_last_hour', feed.pk, int(datetime.datetime.now().strftime('%s')))
if options['force']:
feed.sync_redis()
if not feed or original_feed_id != feed.pk:
logging.info(" ---> ~FRFeed changed id, removing %s from tasked_feeds queue..." % original_feed_id)
r.zrem('tasked_feeds', original_feed_id)
r.zrem('error_feeds', original_feed_id)
if feed:
r.zrem('tasked_feeds', feed.pk)
r.zrem('error_feeds', feed.pk)
return feed
@ -811,7 +840,8 @@ class Feed(models.Model):
def add_update_stories(self, stories, existing_stories, verbose=False):
ret_values = dict(new=0, updated=0, same=0, error=0)
error_count = self.error_count
if settings.DEBUG or verbose:
logging.debug(" ---> [%-30s] ~FBChecking ~SB%s~SN new/updated against ~SB%s~SN stories" % (
self.title[:30],
@ -823,7 +853,10 @@ class Feed(models.Model):
continue
story_content = story.get('story_content')
story_content = strip_comments(story_content)
if error_count:
story_content = strip_comments__lxml(story_content)
else:
story_content = strip_comments(story_content)
story_tags = self.get_tags(story)
story_link = self.get_permalink(story)
@ -1260,7 +1293,11 @@ class Feed(models.Model):
total = max(10, int(updates_per_day_delay + subscriber_bonus + slow_punishment))
if self.active_premium_subscribers >= 3:
total = min(total, 60) # 1 hour minimum for premiums
total = min(total, 3*60) # 1 hour minimum for premiums
elif self.active_premium_subscribers >= 2:
total = min(total, 12*60)
elif self.active_premium_subscribers >= 1:
total = min(total, 24*60)
if self.is_push:
total = total * 20
@ -1283,14 +1320,15 @@ class Feed(models.Model):
self.stories_last_month))
random_factor = random.randint(0, total) / 4
return total, random_factor*8
return total, random_factor*4
def set_next_scheduled_update(self, verbose=False, skip_scheduling=False):
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
total, random_factor = self.get_next_scheduled_update(force=True, verbose=verbose)
error_count = self.error_count
if self.errors_since_good:
total = total * self.errors_since_good
if error_count:
total = total * error_count
if verbose:
logging.debug(' ---> [%-30s] ~FBScheduling feed fetch geometrically: '
'~SB%s errors. Time: %s min' % (
@ -1299,16 +1337,23 @@ class Feed(models.Model):
next_scheduled_update = datetime.datetime.utcnow() + datetime.timedelta(
minutes = total + random_factor)
self.min_to_decay = total
if not skip_scheduling and self.active_subscribers >= 1:
self.next_scheduled_update = next_scheduled_update
r.zadd('scheduled_updates', self.pk, self.next_scheduled_update.strftime('%s'))
r.zrem('tasked_feeds', self.pk)
r.srem('queued_feeds', self.pk)
self.save()
@property
def error_count(self):
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
fetch_errors = int(r.zscore('error_feeds', self.pk) or 0)
return fetch_errors + self.errors_since_good
def schedule_feed_fetch_immediately(self, verbose=True):
r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
if verbose:
@ -1574,6 +1619,8 @@ class MStory(mongo.Document):
if self.id and self.story_date > DAYS_OF_UNREAD:
r.sadd('F:%s' % self.story_feed_id, self.id)
r.zadd('zF:%s' % self.story_feed_id, self.id, time.mktime(self.story_date.timetuple()))
r.expire('F:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
r.expire('zF:%s' % self.story_feed_id, settings.DAYS_OF_UNREAD*24*60*60)
def remove_from_redis(self, r=None):
if not r:

19
apps/rss_feeds/tasks.py
View file

@ -26,17 +26,19 @@ class TaskFeeds(Task):
queued_feeds = r.zrangebyscore('scheduled_updates', 0, now_timestamp)
r.zremrangebyscore('scheduled_updates', 0, now_timestamp)
r.sadd('queued_feeds', *queued_feeds)
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/%s queued/scheduled)" % (
logging.debug(" ---> ~SN~FBQueuing ~SB%s~SN stale feeds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
len(queued_feeds),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
r.zcard('scheduled_updates')))
# Regular feeds
if tasked_feeds_size < 50000:
if tasked_feeds_size < 1000:
feeds = r.srandmember('queued_feeds', 1000)
Feed.task_feeds(feeds, verbose=True)
active_count = len(feeds)
else:
logging.debug(" ---> ~SN~FBToo many tasked feeds. ~SB%s~SN tasked." % tasked_feeds_size)
active_count = 0
cp1 = time.time()
@ -55,7 +57,11 @@ class TaskFeeds(Task):
inactive_count = len(old_tasked_feeds)
if inactive_count:
r.zremrangebyscore('tasked_feeds', 0, hours_ago)
r.sadd('queued_feeds', *old_tasked_feeds)
# r.sadd('queued_feeds', *old_tasked_feeds)
for feed_id in old_tasked_feeds:
r.zincrby('error_feeds', feed_id, 1)
feed = Feed.get_by_id(feed_id)
feed.set_next_scheduled_update()
logging.debug(" ---> ~SN~FBRe-queuing ~SB%s~SN dropped feeds (~SB%s/%s~SN queued/tasked)" % (
inactive_count,
r.scard('queued_feeds'),
@ -85,7 +91,7 @@ class TaskFeeds(Task):
Feed.task_feeds(refresh_feeds, verbose=False)
Feed.task_feeds(old_feeds, verbose=False)
logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~SB%s~SN/%s tasked/queued/scheduled)" % (
logging.debug(" ---> ~SN~FBTasking took ~SB%s~SN seconds (~SB%s~SN/~FG%s~FB~SN/%s tasked/queued/scheduled)" % (
int((time.time() - start)),
r.zcard('tasked_feeds'),
r.scard('queued_feeds'),
@ -106,7 +112,6 @@ class UpdateFeeds(Task):
compute_scores = bool(mongodb_replication_lag < 10)
options = {
'fake': bool(MStatistics.get('fake_fetch')),
'quick': float(MStatistics.get('quick_fetch', 0)),
'compute_scores': compute_scores,
'mongodb_replication_lag': mongodb_replication_lag,
@ -133,9 +138,7 @@ class NewFeeds(Task):
if not isinstance(feed_pks, list):
feed_pks = [feed_pks]
options = {
'force': True,
}
options = {}
for feed_pk in feed_pks:
feed = Feed.get_by_id(feed_pk)
feed.update(options=options)

35
apps/rss_feeds/views.py
View file

@ -1,4 +1,5 @@
import datetime
from urlparse import urlparse
from utils import log as logging
from django.shortcuts import get_object_or_404, render_to_response
from django.views.decorators.http import condition
@ -74,29 +75,24 @@ def feed_autocomplete(request):
query = request.GET.get('term')
version = int(request.GET.get('v', 1))
if True or not user.profile.is_premium:
return dict(code=-1, message="Overloaded, no autocomplete results.", feeds=[], term=query)
# if True or not user.profile.is_premium:
# return dict(code=-1, message="Overloaded, no autocomplete results.", feeds=[], term=query)
if not query:
return dict(code=-1, message="Specify a search 'term'.", feeds=[], term=query)
feeds = []
for field in ['feed_address', 'feed_title', 'feed_link']:
if not feeds:
feeds = Feed.objects.filter(**{
'%s__icontains' % field: query,
'num_subscribers__gt': 1,
'branch_from_feed__isnull': True,
}).exclude(
Q(**{'%s__icontains' % field: 'token'}) |
Q(**{'%s__icontains' % field: 'private'})
).only(
'id',
'feed_title',
'feed_address',
'num_subscribers'
).select_related("data").order_by('-num_subscribers')[:5]
if '.' in query:
try:
parts = urlparse(query)
if not parts.hostname and not query.startswith('http'):
parts = urlparse('http://%s' % query)
if parts.hostname:
query = parts.hostname
except:
logging.user(request, "~FGAdd search, could not parse url in ~FR%s" % query)
feed_ids = Feed.autocomplete(query)
feeds = [Feed.get_by_id(feed_id) for feed_id in feed_ids]
feeds = [{
'id': feed.pk,
'value': feed.feed_address,
@ -104,6 +100,7 @@ def feed_autocomplete(request):
'tagline': feed.data and feed.data.feed_tagline,
'num_subscribers': feed.num_subscribers,
} for feed in feeds]
feeds = sorted(feeds, key=lambda f: -1 * f['num_subscribers'])
feed_ids = [f['id'] for f in feeds]
feed_icons = dict((icon.feed_id, icon) for icon in MFeedIcon.objects.filter(feed_id__in=feed_ids))

2
config/redis.conf
View file

@ -555,5 +555,5 @@ client-output-buffer-limit pubsub 32mb 8mb 60
# to customize a few per-server settings. Include files can include
# other files, so use this wisely.
#
# include /path/to/local.conf
include /etc/redis_server.conf
# include /path/to/other.conf

0
config/redis_master.conf Normal file
View file

1
config/redis_slave.conf Normal file
View file

@ -0,0 +1 @@
slaveof db10 6379

18
fabfile.py vendored
View file

@ -337,6 +337,8 @@ def setup_db(engine=None, skip_common=False):
setup_mongo()
elif engine == "redis":
setup_redis()
elif engine == "redis_slave":
setup_redis(slave=True)
setup_gunicorn(supervisor=False)
setup_db_munin()
@ -456,10 +458,10 @@ def setup_imaging():
def setup_supervisor():
sudo('apt-get -y install supervisor')
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf')
sudo('/etc/init.d/supervisord stop')
put('config/supervisord.conf', '/etc/supervisor/supervisord.conf', use_sudo=True)
sudo('/etc/init.d/supervisor stop')
sudo('sleep 2')
sudo('/etc/init.d/supervisord start')
sudo('/etc/init.d/supervisor start')
# @parallel
def setup_hosts():
@ -563,7 +565,7 @@ def setup_sudoers(user=None):
sudo('su - root -c "echo \\\\"%s ALL=(ALL) NOPASSWD: ALL\\\\" >> /etc/sudoers"' % (user or env.user))
def setup_nginx():
NGINX_VERSION = '1.2.2'
NGINX_VERSION = '1.2.8'
with cd(env.VENDOR_PATH), settings(warn_only=True):
sudo("groupadd nginx")
sudo("useradd -g nginx -d /var/www/htdocs -s /bin/false nginx")
@ -834,8 +836,8 @@ def setup_mongo_mms():
sudo('supervisorctl update')
def setup_redis():
redis_version = '2.6.11'
def setup_redis(slave=False):
redis_version = '2.6.12'
with cd(env.VENDOR_PATH):
run('wget http://redis.googlecode.com/files/redis-%s.tar.gz' % redis_version)
run('tar -xzf redis-%s.tar.gz' % redis_version)
@ -845,6 +847,10 @@ def setup_redis():
put('config/redis-init', '/etc/init.d/redis', use_sudo=True)
sudo('chmod u+x /etc/init.d/redis')
put('config/redis.conf', '/etc/redis.conf', use_sudo=True)
if slave:
put('config/redis_slave.conf', '/etc/redis_server.conf', use_sudo=True)
else:
put('config/redis_master.conf', '/etc/redis_server.conf', use_sudo=True)
sudo('mkdir -p /var/lib/redis')
sudo('update-rc.d redis defaults')
sudo('/etc/init.d/redis stop')

3
media/android/NewsBlur/src/com/newsblur/activity/AddFacebook.java
View file

@ -5,10 +5,9 @@ import android.text.TextUtils;
import android.webkit.WebView;
import android.webkit.WebViewClient;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
public class AddFacebook extends SherlockFragmentActivity {
public class AddFacebook extends NbFragmentActivity {
public static final int FACEBOOK_AUTHED = 0x21;
private WebView webview;

3
media/android/NewsBlur/src/com/newsblur/activity/AddFollow.java
View file

@ -8,11 +8,10 @@ import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
import com.newsblur.fragment.AddFollowFragment;
public class AddFollow extends SherlockFragmentActivity {
public class AddFollow extends NbFragmentActivity {
private FragmentManager fragmentManager;
private String currentTag = "addFollowFragment";

3
media/android/NewsBlur/src/com/newsblur/activity/AddSites.java
View file

@ -11,12 +11,11 @@ import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
import com.newsblur.fragment.AddSitesListFragment;
import com.newsblur.network.APIManager;
public class AddSites extends SherlockFragmentActivity {
public class AddSites extends NbFragmentActivity {
private FragmentManager fragmentManager;
private String currentTag = "addsitesFragment";

3
media/android/NewsBlur/src/com/newsblur/activity/AddSocial.java
View file

@ -8,11 +8,10 @@ import android.view.View;
import android.view.View.OnClickListener;
import android.widget.Button;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
import com.newsblur.fragment.AddSocialFragment;
public class AddSocial extends SherlockFragmentActivity {
public class AddSocial extends NbFragmentActivity {
private FragmentManager fragmentManager;
private String currentTag = "addSocialFragment";

3
media/android/NewsBlur/src/com/newsblur/activity/AddTwitter.java
View file

@ -5,10 +5,9 @@ import android.text.TextUtils;
import android.webkit.WebView;
import android.webkit.WebViewClient;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
public class AddTwitter extends SherlockFragmentActivity {
public class AddTwitter extends NbFragmentActivity {
public static final int TWITTER_AUTHED = 0x20;
private WebView webview;

3
media/android/NewsBlur/src/com/newsblur/activity/FeedSearch.java
View file

@ -1,7 +1,6 @@
package com.newsblur.activity;
import com.actionbarsherlock.app.SherlockFragmentActivity;
public class FeedSearch extends SherlockFragmentActivity {
public class FeedSearch extends NbFragmentActivity {
}

3
media/android/NewsBlur/src/com/newsblur/activity/ImportFeeds.java
View file

@ -5,10 +5,9 @@ import android.text.TextUtils;
import android.webkit.WebView;
import android.webkit.WebViewClient;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
public class ImportFeeds extends SherlockFragmentActivity {
public class ImportFeeds extends NbFragmentActivity {
private WebView webContainer;

3
media/android/NewsBlur/src/com/newsblur/activity/ItemsList.java
View file

@ -5,7 +5,6 @@ import android.os.Bundle;
import android.support.v4.app.FragmentManager;
import android.util.Log;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuItem;
import com.actionbarsherlock.view.Window;
@ -15,7 +14,7 @@ import com.newsblur.fragment.ItemListFragment;
import com.newsblur.fragment.SyncUpdateFragment;
import com.newsblur.view.StateToggleButton.StateChangedListener;
public abstract class ItemsList extends SherlockFragmentActivity implements SyncUpdateFragment.SyncUpdateFragmentInterface, StateChangedListener {
public abstract class ItemsList extends NbFragmentActivity implements SyncUpdateFragment.SyncUpdateFragmentInterface, StateChangedListener {
public static final String EXTRA_STATE = "currentIntelligenceState";
public static final String EXTRA_BLURBLOG_USERNAME = "blurblogName";

3
media/android/NewsBlur/src/com/newsblur/activity/Login.java
View file

@ -1,5 +1,6 @@
package com.newsblur.activity;
import android.content.Context;
import android.content.Intent;
import android.content.SharedPreferences;
import android.os.Bundle;
@ -34,7 +35,7 @@ public class Login extends FragmentActivity {
}
private void preferenceCheck() {
final SharedPreferences preferences = getSharedPreferences(PrefConstants.PREFERENCES, 0);
final SharedPreferences preferences = getSharedPreferences(PrefConstants.PREFERENCES, Context.MODE_PRIVATE);
if (preferences.getString(PrefConstants.PREF_COOKIE, null) != null) {
final Intent mainIntent = new Intent(this, Main.class);
startActivity(mainIntent);

3
media/android/NewsBlur/src/com/newsblur/activity/Main.java
View file

@ -7,7 +7,6 @@ import android.support.v4.app.FragmentManager;
import android.util.Log;
import com.actionbarsherlock.app.ActionBar;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuInflater;
import com.actionbarsherlock.view.MenuItem;
@ -19,7 +18,7 @@ import com.newsblur.fragment.SyncUpdateFragment;
import com.newsblur.service.SyncService;
import com.newsblur.view.StateToggleButton.StateChangedListener;
public class Main extends SherlockFragmentActivity implements StateChangedListener, SyncUpdateFragment.SyncUpdateFragmentInterface {
public class Main extends NbFragmentActivity implements StateChangedListener, SyncUpdateFragment.SyncUpdateFragmentInterface {
private ActionBar actionBar;
private FolderListFragment folderFeedList;

48
media/android/NewsBlur/src/com/newsblur/activity/NbFragmentActivity.java Normal file
View file

@ -0,0 +1,48 @@
package com.newsblur.activity;
import android.os.Bundle;
import android.util.Log;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.util.PrefsUtils;
public class NbFragmentActivity extends SherlockFragmentActivity {
private final static String UNIQUE_LOGIN_KEY = "uniqueLoginKey";
private String uniqueLoginKey;
@Override
protected void onCreate(Bundle bundle) {
super.onCreate(bundle);
if(bundle == null) {
uniqueLoginKey = PrefsUtils.getUniqueLoginKey(this);
} else {
uniqueLoginKey = bundle.getString(UNIQUE_LOGIN_KEY);
}
finishIfNotLoggedIn();
}
@Override
protected void onResume() {
super.onResume();
finishIfNotLoggedIn();
}
protected void finishIfNotLoggedIn() {
String currentLoginKey = PrefsUtils.getUniqueLoginKey(this);
if(currentLoginKey == null || !currentLoginKey.equals(uniqueLoginKey)) {
Log.d( this.getClass().getName(), "This activity was for a different login. finishing it.");
finish();
}
}
@Override
public void onSaveInstanceState(Bundle savedInstanceState) {
savedInstanceState.putString(UNIQUE_LOGIN_KEY, uniqueLoginKey);
super.onSaveInstanceState(savedInstanceState);
}
}

3
media/android/NewsBlur/src/com/newsblur/activity/Profile.java
View file

@ -6,7 +6,6 @@ import android.support.v4.app.FragmentManager;
import android.support.v4.app.FragmentTransaction;
import android.text.TextUtils;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.actionbarsherlock.view.MenuItem;
import com.newsblur.R;
import com.newsblur.domain.UserDetails;
@ -17,7 +16,7 @@ import com.newsblur.network.domain.ActivitiesResponse;
import com.newsblur.network.domain.ProfileResponse;
import com.newsblur.util.PrefsUtils;
public class Profile extends SherlockFragmentActivity {
public class Profile extends NbFragmentActivity {
private FragmentManager fragmentManager;
private String detailsTag = "details";

5
media/android/NewsBlur/src/com/newsblur/activity/Reading.java
View file

@ -10,7 +10,6 @@ import android.content.Intent;
import android.content.OperationApplicationException;
import android.database.Cursor;
import android.net.Uri;
import android.os.AsyncTask;
import android.os.Bundle;
import android.os.RemoteException;
import android.support.v4.app.DialogFragment;
@ -21,9 +20,7 @@ import android.text.TextUtils;
import android.util.Log;
import android.widget.SeekBar;
import android.widget.SeekBar.OnSeekBarChangeListener;
import android.widget.Toast;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuInflater;
import com.actionbarsherlock.view.MenuItem;
@ -44,7 +41,7 @@ import com.newsblur.util.PrefConstants;
import com.newsblur.util.PrefsUtils;
import com.newsblur.util.UIUtils;
public abstract class Reading extends SherlockFragmentActivity implements OnPageChangeListener, SyncUpdateFragment.SyncUpdateFragmentInterface, OnSeekBarChangeListener {
public abstract class Reading extends NbFragmentActivity implements OnPageChangeListener, SyncUpdateFragment.SyncUpdateFragmentInterface, OnSeekBarChangeListener {
public static final String EXTRA_FEED = "feed_selected";
public static final String TAG = "ReadingActivity";

4
media/android/NewsBlur/src/com/newsblur/activity/RegisterProgress.java
View file

@ -3,13 +3,11 @@ package com.newsblur.activity;
import android.os.Bundle;
import android.support.v4.app.FragmentManager;
import android.support.v4.app.FragmentTransaction;
import android.util.Log;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.newsblur.R;
import com.newsblur.fragment.RegisterProgressFragment;
public class RegisterProgress extends SherlockFragmentActivity {
public class RegisterProgress extends NbFragmentActivity {
private FragmentManager fragmentManager;
private String currentTag = "fragment";

3
media/android/NewsBlur/src/com/newsblur/activity/SearchForFeeds.java
View file

@ -13,7 +13,6 @@ import android.widget.ListView;
import android.widget.TextView;
import android.widget.Toast;
import com.actionbarsherlock.app.SherlockFragmentActivity;
import com.actionbarsherlock.view.Menu;
import com.actionbarsherlock.view.MenuInflater;
import com.actionbarsherlock.view.MenuItem;
@ -24,7 +23,7 @@ import com.newsblur.fragment.AddFeedFragment;
import com.newsblur.network.SearchAsyncTaskLoader;
import com.newsblur.network.SearchLoaderResponse;
public class SearchForFeeds extends SherlockFragmentActivity implements LoaderCallbacks<SearchLoaderResponse>, OnItemClickListener {
public class SearchForFeeds extends NbFragmentActivity implements LoaderCallbacks<SearchLoaderResponse>, OnItemClickListener {
private static final int LOADER_TWITTER_SEARCH = 0x01;
private ListView resultsList;
private Loader<SearchLoaderResponse> searchLoader;

5
media/android/NewsBlur/src/com/newsblur/fragment/LogoutDialogFragment.java
View file

@ -14,13 +14,12 @@ import android.widget.TextView;
import com.newsblur.R;
import com.newsblur.activity.Login;
import com.newsblur.database.BlurDatabase;
import com.newsblur.network.APIManager;
import com.newsblur.util.PrefConstants;
import com.newsblur.util.PrefsUtils;
public class LogoutDialogFragment extends DialogFragment {
protected static final String TAG = "LogoutDialogFragment";
private APIManager apiManager;
@Override
public void onCreate(Bundle savedInstanceState) {
@ -31,7 +30,6 @@ public class LogoutDialogFragment extends DialogFragment {
@Override
public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle bundle) {
apiManager = new APIManager(getActivity());
View v = inflater.inflate(R.layout.fragment_logout_dialog, container, false);
final TextView message = (TextView) v.findViewById(R.id.dialog_message);
message.setText(getActivity().getResources().getString(R.string.logout_warning));
@ -45,6 +43,7 @@ public class LogoutDialogFragment extends DialogFragment {
BlurDatabase databaseHelper = new BlurDatabase(getActivity().getApplicationContext());
databaseHelper.dropAndRecreateTables();
PrefsUtils.clearLogin(getActivity());
Intent i = new Intent(getActivity(), Login.class);
i.setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
startActivity(i);

4
media/android/NewsBlur/src/com/newsblur/network/APIManager.java
View file

@ -75,7 +75,7 @@ public class APIManager {
final APIResponse response = client.post(APIConstants.URL_LOGIN, values);
if (response.responseCode == HttpStatus.SC_OK && !response.hasRedirected) {
LoginResponse loginResponse = gson.fromJson(response.responseString, LoginResponse.class);
PrefsUtils.saveCookie(context, response.cookie);
PrefsUtils.saveLogin(context, username, response.cookie);
return loginResponse;
} else {
return new LoginResponse();
@ -185,7 +185,7 @@ public class APIManager {
final APIResponse response = client.post(APIConstants.URL_SIGNUP, values);
if (response.responseCode == HttpStatus.SC_OK && !response.hasRedirected) {
LoginResponse loginResponse = gson.fromJson(response.responseString, LoginResponse.class);
PrefsUtils.saveCookie(context, response.cookie);
PrefsUtils.saveLogin(context, username, response.cookie);
CookieSyncManager.createInstance(context.getApplicationContext());
CookieManager cookieManager = CookieManager.getInstance();

1
media/android/NewsBlur/src/com/newsblur/util/PrefConstants.java
View file

@ -4,6 +4,7 @@ public class PrefConstants {
public static final String PREFERENCES = "preferences";
public static final String PREF_COOKIE = "login_cookie";
public static final String PREF_UNIQUE_LOGIN = "login_unique";
public final static String USER_USERNAME = "username";
public final static String USER_WEBSITE = "website";

24
media/android/NewsBlur/src/com/newsblur/util/PrefsUtils.java
View file

@ -18,13 +18,35 @@ import com.newsblur.domain.UserDetails;
public class PrefsUtils {
public static void saveCookie(final Context context, final String cookie) {
public static void saveLogin(final Context context, final String userName, final String cookie) {
final SharedPreferences preferences = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
final Editor edit = preferences.edit();
edit.putString(PrefConstants.PREF_COOKIE, cookie);
edit.putString(PrefConstants.PREF_UNIQUE_LOGIN, userName + "_" + System.currentTimeMillis());
edit.commit();
}
/**
* Removes the current login session (i.e. logout)
*/
public static void clearLogin(final Context context) {
final SharedPreferences preferences = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
final Editor edit = preferences.edit();
edit.putString(PrefConstants.PREF_COOKIE, null);
edit.putString(PrefConstants.PREF_UNIQUE_LOGIN, null);
edit.commit();
}
/**
* Retrieves the current unique login key. This key will be unique for each
* login. If this login key doesn't match the login key you have then assume
* the user is logged out
*/
public static String getUniqueLoginKey(final Context context) {
final SharedPreferences preferences = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
return preferences.getString(PrefConstants.PREF_UNIQUE_LOGIN, null);
}
public static void saveUserDetails(final Context context, final UserDetails profile) {
final SharedPreferences preferences = context.getSharedPreferences(PrefConstants.PREFERENCES, 0);
final Editor edit = preferences.edit();

2
media/js/newsblur/reader/reader_goodies.js
View file

@ -182,7 +182,7 @@ NEWSBLUR.ReaderGoodies.prototype = {
$.targetIs(e, { tagSelector: '.NB-goodies-chrome-link' }, function($t, $p) {
e.preventDefault();
window.location.href = 'https://chrome.google.com/webstore/detail/gchdledhagjbhhodjjhiclbnaioljomj';
window.location.href = 'https://chrome.google.com/webstore/detail/rss-subscription-extensio/bmjffnfcokiodbeiamclanljnaheeoke';
});
$.targetIs(e, { tagSelector: '.NB-goodies-custom-input' }, function($t, $p) {

2
settings.py
View file

@ -549,6 +549,8 @@ REDIS_ANALYTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=2)
REDIS_STATISTICS_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=3)
REDIS_FEED_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=4)
REDIS_SESSION_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=5)
# REDIS_CACHE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=6) # Duped in CACHES
REDIS_AUTOCOMPLETE_POOL = redis.ConnectionPool(host=REDIS['host'], port=6379, db=7)
JAMMIT = jammit.JammitAssets(NEWSBLUR_DIR)

2
templates/maintenance_off.html
View file

@ -77,7 +77,7 @@
<img src="/media/img/logo_512.png" class="logo">
<h1>NewsBlur is in <span class="error404">maintenance mode</span></h1>
<div class="description">
<p>This will take approximately 5 minutes. I'm upgrading PostgreSQL and re-building indexes for optimum performance. Tonight's a rough night, but hopefully this will be the last of it.</p>
<p>This will take approximately 5-10 minutes. I'm upgrading Redis, which is throwing faults due to memory issues.</p>
<p>To pass the time, go surf <a href="http://mlkshk.com/popular">MLKSHK's popular page</a>.</p>
<p></p>
</div>

9
utils/feed_fetcher.py
View file

@ -369,10 +369,15 @@ class Dispatcher:
feed.known_good = True
feed.fetched_once = True
feed = feed.save()
if random.random() <= 0.02:
if self.options['force'] or random.random() <= 0.02:
logging.debug(' ---> [%-30s] ~FBPerforming feed cleanup...' % (feed.title[:30],))
start_cleanup = time.time()
feed.sync_redis()
cp1 = time.time() - start_cleanup
MUserStory.delete_old_stories(feed_id=feed.pk)
MUserStory.sync_all_redis(feed_id=feed.pk)
cp2 = time.time() - cp1 - start_cleanup
# MUserStory.sync_all_redis(feed_id=feed.pk)
logging.debug(' ---> [%-30s] ~FBDone with feed cleanup. Took %.4s+%.4s+%.4s=~SB%.4s~SN sec.' % (feed.title[:30], cp1, cp2, time.time() - cp2 - cp1 - start_cleanup, time.time() - start_cleanup))
try:
self.count_unreads_for_subscribers(feed)
except TimeoutError:

2
utils/munin/newsblur_updates.py
View file

@ -14,6 +14,7 @@ class NBMuninGraph(MuninGraph):
'update_queue.label': 'Queued Feeds',
'feeds_fetched.label': 'Fetched feeds last hour',
'tasked_feeds.label': 'Tasked Feeds',
'error_feeds.label': 'Error Feeds',
'celery_update_feeds.label': 'Celery - Update Feeds',
'celery_new_feeds.label': 'Celery - New Feeds',
'celery_push_feeds.label': 'Celery - Push Feeds',
@ -30,6 +31,7 @@ class NBMuninGraph(MuninGraph):
'update_queue': r.scard("queued_feeds"),
'feeds_fetched': r.zcard("fetched_feeds_last_hour"),
'tasked_feeds': r.zcard("tasked_feeds"),
'error_feeds': r.zcard("error_feeds"),
'celery_update_feeds': r.llen("update_feeds"),
'celery_new_feeds': r.llen("new_feeds"),
'celery_push_feeds': r.llen("push_feeds"),

23
utils/story_functions.py
View file

@ -15,7 +15,8 @@ from utils.tornado_escape import linkify as linkify_tornado
from utils.tornado_escape import xhtml_unescape as xhtml_unescape_tornado
from vendor import reseekfile
COMMENTS_RE = re.compile('\<![ \r\n\t]*(--([^\-]|[\r\n]|-[^\-])*--[ \r\n\t]*)\>')
# COMMENTS_RE = re.compile('\<![ \r\n\t]*(--([^\-]|[\r\n]|-[^\-])*--[ \r\n\t]*)\>')
COMMENTS_RE = re.compile('\<!--.*?--\>')
def story_score(story, bottom_delta=None):
# A) Date - Assumes story is unread and within unread range
@ -83,6 +84,9 @@ def pre_process_story(entry):
else:
entry['published'] = datetime.datetime.utcnow()
if entry['published'] > datetime.datetime.now() + datetime.timedelta(days=1):
entry['published'] = datetime.datetime.now()
# entry_link = entry.get('link') or ''
# protocol_index = entry_link.find("://")
# if protocol_index != -1:
@ -194,8 +198,21 @@ def strip_tags(html):
def strip_comments(html_string):
return COMMENTS_RE.sub('', html_string)
def strip_comments__lxml2(html_string=""):
if not html_string: return html_string
tree = lxml.html.fromstring(html_string)
comments = tree.xpath('//comment()')
for c in comments:
p = c.getparent()
p.remove(c)
return lxml.etree.tostring(tree)
def strip_comments__lxml(html_string=""):
if not html_string: return html_string
def strip_comments__lxml(html_string):
params = {
'comments': True,
'scripts': False,
@ -222,7 +239,7 @@ def strip_comments__lxml(html_string):
return lxml.etree.tostring(clean_html)
except XMLSyntaxError:
return html_string
def linkify(*args, **kwargs):
return xhtml_unescape_tornado(linkify_tornado(*args, **kwargs))

1
vendor/redis_completion/__init__.py vendored Executable file
View file

@ -0,0 +1 @@
from redis_completion.engine import RedisEngine

228
vendor/redis_completion/engine.py vendored Executable file
View file

@ -0,0 +1,228 @@
try:
import simplejson as json
except ImportError:
import json
import re
from redis import Redis
from redis_completion.stop_words import STOP_WORDS as _STOP_WORDS
# aggressive stop words will be better when the length of the document is longer
AGGRESSIVE_STOP_WORDS = _STOP_WORDS
# default stop words should work fine for titles and things like that
DEFAULT_STOP_WORDS = set(['a', 'an', 'of', 'the'])
class RedisEngine(object):
"""
References
----------
http://antirez.com/post/autocomplete-with-redis.html
http://stackoverflow.com/questions/1958005/redis-autocomplete/1966188#1966188
http://patshaughnessy.net/2011/11/29/two-ways-of-using-redis-to-build-a-nosql-autocomplete-search-index
"""
def __init__(self, prefix='ac', stop_words=None, cache_timeout=300, **conn_kwargs):
self.prefix = prefix
self.stop_words = (stop_words is None) and DEFAULT_STOP_WORDS or stop_words
self.conn_kwargs = conn_kwargs
self.client = self.get_client()
self.cache_timeout = cache_timeout
self.boost_key = '%s:b' % self.prefix
self.data_key = '%s:d' % self.prefix
self.title_key = '%s:t' % self.prefix
self.search_key = lambda k: '%s:s:%s' % (self.prefix, k)
self.cache_key = lambda pk, bk: '%s:c:%s:%s' % (self.prefix, pk, bk)
self.kcombine = lambda _id, _type: str(_id)
self.ksplit = lambda k: k
def get_client(self):
return Redis(**self.conn_kwargs)
def score_key(self, k, max_size=20):
k_len = len(k)
a = ord('a') - 2
score = 0
for i in range(max_size):
if i < k_len:
c = (ord(k[i]) - a)
if c < 2 or c > 27:
c = 1
else:
c = 1
score += c*(27**(max_size-i))
return score
def clean_phrase(self, phrase):
phrase = re.sub('[^a-z0-9_\-\s]', '', phrase.lower())
return [w for w in phrase.split() if w not in self.stop_words]
def create_key(self, phrase):
return ' '.join(self.clean_phrase(phrase))
def autocomplete_keys(self, w):
for i in range(1, len(w)):
yield w[:i]
yield w
def flush(self, everything=False, batch_size=1000):
if everything:
return self.client.flushdb()
# this could be expensive :-(
keys = self.client.keys('%s:*' % self.prefix)
# batch keys
for i in range(0, len(keys), batch_size):
self.client.delete(*keys[i:i+batch_size])
def store(self, obj_id, title=None, data=None, obj_type=None, check_exist=True):
if title is None:
title = obj_id
if data is None:
data = title
title_score = self.score_key(self.create_key(title))
combined_id = self.kcombine(obj_id, obj_type or '')
if check_exist and self.exists(obj_id, obj_type):
stored_title = self.client.hget(self.title_key, combined_id)
# if the stored title is the same, we can simply update the data key
# since everything else will have stayed the same
if stored_title == title:
self.client.hset(self.data_key, combined_id, data)
return
else:
self.remove(obj_id, obj_type)
pipe = self.client.pipeline()
pipe.hset(self.data_key, combined_id, data)
pipe.hset(self.title_key, combined_id, title)
for word in self.clean_phrase(title):
for partial_key in self.autocomplete_keys(word):
pipe.zadd(self.search_key(partial_key), combined_id, title_score)
pipe.execute()
def store_json(self, obj_id, title, data_dict, obj_type=None):
return self.store(obj_id, title, json.dumps(data_dict), obj_type)
def remove(self, obj_id, obj_type=None):
obj_id = self.kcombine(obj_id, obj_type or '')
title = self.client.hget(self.title_key, obj_id) or ''
keys = []
for word in self.clean_phrase(title):
for partial_key in self.autocomplete_keys(word):
key = self.search_key(partial_key)
if not self.client.zrange(key, 1, 2):
self.client.delete(key)
else:
self.client.zrem(key, obj_id)
self.client.hdel(self.data_key, obj_id)
self.client.hdel(self.title_key, obj_id)
self.client.hdel(self.boost_key, obj_id)
def boost(self, obj_id, multiplier=1.1, negative=False):
# take the existing boost for this item and increase it by the multiplier
current = self.client.hget(self.boost_key, obj_id)
current_f = float(current or 1.0)
if negative:
multiplier = 1 / multiplier
self.client.hset(self.boost_key, obj_id, current_f * multiplier)
def exists(self, obj_id, obj_type=None):
obj_id = self.kcombine(obj_id, obj_type or '')
return self.client.hexists(self.data_key, obj_id)
def get_cache_key(self, phrases, boosts):
if boosts:
boost_key = '|'.join('%s:%s' % (k, v) for k, v in sorted(boosts.items()))
else:
boost_key = ''
phrase_key = '|'.join(phrases)
return self.cache_key(phrase_key, boost_key)
def _process_ids(self, id_list, limit, filters, mappers):
ct = 0
data = []
for raw_id in id_list:
# raw_data = self.client.hget(self.data_key, raw_id)
raw_data = raw_id
if not raw_data:
continue
if mappers:
for m in mappers:
raw_data = m(raw_data)
if filters:
passes = True
for f in filters:
if not f(raw_data):
passes = False
break
if not passes:
continue
data.append(raw_data)
ct += 1
if limit and ct == limit:
break
return data
def search(self, phrase, limit=None, filters=None, mappers=None, boosts=None, autoboost=False):
cleaned = self.clean_phrase(phrase)
if not cleaned:
return []
if autoboost:
boosts = boosts or {}
stored = self.client.hgetall(self.boost_key)
for obj_id in stored:
if obj_id not in boosts:
boosts[obj_id] = float(stored[obj_id])
if len(cleaned) == 1 and not boosts:
new_key = self.search_key(cleaned[0])
else:
new_key = self.get_cache_key(cleaned, boosts)
if not self.client.exists(new_key):
# zinterstore also takes {k1: wt1, k2: wt2}
self.client.zinterstore(new_key, map(self.search_key, cleaned))
self.client.expire(new_key, self.cache_timeout)
if boosts:
pipe = self.client.pipeline()
for raw_id, score in self.client.zrange(new_key, 0, -1, withscores=True):
orig_score = score
for part in self.ksplit(raw_id):
if part and part in boosts:
score *= 1 / boosts[part]
if orig_score != score:
pipe.zadd(new_key, raw_id, score)
pipe.execute()
id_list = self.client.zrange(new_key, 0, -1)
# return id_list
return self._process_ids(id_list, limit, filters, mappers)
def search_json(self, phrase, limit=None, filters=None, mappers=None, boosts=None, autoboost=False):
if not mappers:
mappers = []
mappers.insert(0, json.loads)
return self.search(phrase, limit, filters, mappers, boosts, autoboost)

594
vendor/redis_completion/stop_words.py vendored Executable file
View file

@ -0,0 +1,594 @@
words = """a
a's
able
about
above
according
accordingly
across
actually
after
afterwards
again
against
ain't
all
allow
allows
almost
alone
along
already
also
although
always
am
among
amongst
amoungst
amount
an
and
another
any
anybody
anyhow
anyone
anything
anyway
anyways
anywhere
apart
appear
appreciate
appropriate
are
aren't
around
as
aside
ask
asking
associated
at
available
away
awfully
back
be
became
because
become
becomes
becoming
been
before
beforehand
behind
being
believe
below
beside
besides
best
better
between
beyond
bill
both
bottom
brief
but
by
c'mon
c's
call
came
can
can't
cannot
cant
cause
causes
certain
certainly
changes
clearly
co
com
come
comes
computer
con
concerning
consequently
consider
considering
contain
containing
contains
corresponding
could
couldn't
couldnt
course
cry
currently
de
definitely
describe
described
despite
detail
did
didn't
different
do
does
doesn't
doing
don't
done
down
downwards
due
during
each
edu
eg
eight
either
eleven
else
elsewhere
empty
enough
entirely
especially
et
etc
even
ever
every
everybody
everyone
everything
everywhere
ex
exactly
example
except
far
few
fifteen
fifth
fify
fill
find
fire
first
five
followed
following
follows
for
former
formerly
forth
forty
found
four
from
front
full
further
furthermore
get
gets
getting
give
given
gives
go
goes
going
gone
got
gotten
greetings
had
hadn't
happens
hardly
has
hasn't
hasnt
have
haven't
having
he
he's
hello
help
hence
her
here
here's
hereafter
hereby
herein
hereupon
hers
herself
hi
him
himself
his
hither
hopefully
how
howbeit
however
hundred
i
i'd
i'll
i'm
i've
ie
if
ignored
immediate
in
inasmuch
inc
indeed
indicate
indicated
indicates
inner
insofar
instead
interest
into
inward
is
isn't
it
it'd
it'll
it's
its
itself
just
keep
keeps
kept
know
known
knows
last
lately
later
latter
latterly
least
less
lest
let
let's
like
liked
likely
little
look
looking
looks
ltd
made
mainly
many
may
maybe
me
mean
meanwhile
merely
might
mill
mine
more
moreover
most
mostly
move
much
must
my
myself
name
namely
nd
near
nearly
necessary
need
needs
neither
never
nevertheless
new
next
nine
no
nobody
non
none
noone
nor
normally
not
nothing
novel
now
nowhere
obviously
of
off
often
oh
ok
okay
old
on
once
one
ones
only
onto
or
other
others
otherwise
ought
our
ours
ourselves
out
outside
over
overall
own
part
particular
particularly
per
perhaps
placed
please
plus
possible
presumably
probably
provides
put
que
quite
qv
rather
rd
re
really
reasonably
regarding
regardless
regards
relatively
respectively
right
said
same
saw
say
saying
says
second
secondly
see
seeing
seem
seemed
seeming
seems
seen
self
selves
sensible
sent
serious
seriously
seven
several
shall
she
should
shouldn't
show
side
since
sincere
six
sixty
so
some
somebody
somehow
someone
something
sometime
sometimes
somewhat
somewhere
soon
sorry
specified
specify
specifying
still
sub
such
sup
sure
system
t's
take
taken
tell
ten
tends
th
than
thank
thanks
thanx
that
that's
thats
the
their
theirs
them
themselves
then
thence
there
there's
thereafter
thereby
therefore
therein
theres
thereupon
these
they
they'd
they'll
they're
they've
thick
thin
think
third
this
thorough
thoroughly
those
though
three
through
throughout
thru
thus
to
together
too
took
top
toward
towards
tried
tries
truly
try
trying
twelve
twenty
twice
two
un
under
unfortunately
unless
unlikely
until
unto
up
upon
us
use
used
useful
uses
using
usually
value
various
very
via
viz
vs
want
wants
was
wasn't
way
we
we'd
we'll
we're
we've
welcome
well
went
were
weren't
what
what's
whatever
when
whence
whenever
where
where's
whereafter
whereas
whereby
wherein
whereupon
wherever
whether
which
while
whither
who
who's
whoever
whole
whom
whose
why
will
willing
wish
with
within
without
won't
wonder
would
wouldn't
yes
yet
you
you'd
you'll
you're
you've
your
yours
yourself
yourselves
zero"""
STOP_WORDS = set([
w.strip() for w in words.splitlines() if w
])

277
vendor/redis_completion/tests.py vendored Executable file
View file

@ -0,0 +1,277 @@
import random
from unittest import TestCase
from redis_completion.engine import RedisEngine
stop_words = set(['a', 'an', 'the', 'of'])
class RedisCompletionTestCase(TestCase):
def setUp(self):
self.engine = self.get_engine()
self.engine.flush()
def get_engine(self):
return RedisEngine(prefix='testac', db=15)
def store_data(self, id=None):
test_data = (
(1, 'testing python'),
(2, 'testing python code'),
(3, 'web testing python code'),
(4, 'unit tests with python'),
)
for obj_id, title in test_data:
if id is None or id == obj_id:
self.engine.store_json(obj_id, title, {
'obj_id': obj_id,
'title': title,
'secret': obj_id % 2 == 0 and 'derp' or 'herp',
})
def sort_results(self, r):
return sorted(r, key=lambda i:i['obj_id'])
def test_search(self):
self.store_data()
results = self.engine.search_json('testing python')
self.assertEqual(self.sort_results(results), [
{'obj_id': 1, 'title': 'testing python', 'secret': 'herp'},
{'obj_id': 2, 'title': 'testing python code', 'secret': 'derp'},
{'obj_id': 3, 'title': 'web testing python code', 'secret': 'herp'},
])
results = self.engine.search_json('test')
self.assertEqual(self.sort_results(results), [
{'obj_id': 1, 'title': 'testing python', 'secret': 'herp'},
{'obj_id': 2, 'title': 'testing python code', 'secret': 'derp'},
{'obj_id': 3, 'title': 'web testing python code', 'secret': 'herp'},
{'obj_id': 4, 'title': 'unit tests with python', 'secret': 'derp'},
])
results = self.engine.search_json('unit')
self.assertEqual(results, [
{'obj_id': 4, 'title': 'unit tests with python', 'secret': 'derp'},
])
results = self.engine.search_json('')
self.assertEqual(results, [])
results = self.engine.search_json('missing')
self.assertEqual(results, [])
def test_boosting(self):
test_data = (
(1, 'test alpha', 't1'),
(2, 'test beta', 't1'),
(3, 'test gamma', 't1'),
(4, 'test delta', 't1'),
(5, 'test alpha', 't2'),
(6, 'test beta', 't2'),
(7, 'test gamma', 't2'),
(8, 'test delta', 't2'),
(9, 'test alpha', 't3'),
(10, 'test beta', 't3'),
(11, 'test gamma', 't3'),
(12, 'test delta', 't3'),
)
for obj_id, title, obj_type in test_data:
self.engine.store_json(obj_id, title, {
'obj_id': obj_id,
'title': title,
}, obj_type)
def assertExpected(results, id_list):
self.assertEqual([r['obj_id'] for r in results], id_list)
results = self.engine.search_json('alp')
assertExpected(results, [1, 5, 9])
results = self.engine.search_json('alp', boosts={'t2': 1.1})
assertExpected(results, [5, 1, 9])
results = self.engine.search_json('test', boosts={'t3': 1.5, 't2': 1.1})
assertExpected(results, [9, 10, 12, 11, 5, 6, 8, 7, 1, 2, 4, 3])
results = self.engine.search_json('alp', boosts={'t1': 0.5})
assertExpected(results, [5, 9, 1])
results = self.engine.search_json('alp', boosts={'t1': 1.5, 't3': 1.6})
assertExpected(results, [9, 1, 5])
results = self.engine.search_json('alp', boosts={'t3': 1.5, '5': 1.6})
assertExpected(results, [5, 9, 1])
def test_autoboost(self):
self.engine.store('t1', 'testing 1')
self.engine.store('t2', 'testing 2')
self.engine.store('t3', 'testing 3')
self.engine.store('t4', 'testing 4')
self.engine.store('t5', 'testing 5')
def assertExpected(results, id_list):
self.assertEqual(results, ['testing %s' % i for i in id_list])
results = self.engine.search('testing', autoboost=True)
assertExpected(results, [1, 2, 3, 4, 5])
self.engine.boost('t3')
results = self.engine.search('testing', autoboost=True)
assertExpected(results, [3, 1, 2, 4, 5])
self.engine.boost('t2')
results = self.engine.search('testing', autoboost=True)
assertExpected(results, [2, 3, 1, 4, 5])
self.engine.boost('t1', negative=True)
results = self.engine.search('testing', autoboost=True)
assertExpected(results, [2, 3, 4, 5, 1])
results = self.engine.search('testing', boosts={'t5': 4.0}, autoboost=True)
assertExpected(results, [5, 2, 3, 4, 1])
results = self.engine.search('testing', boosts={'t3': 1.5}, autoboost=True)
assertExpected(results, [3, 2, 4, 5, 1])
def test_limit(self):
self.store_data()
results = self.engine.search_json('testing', limit=1)
self.assertEqual(results, [
{'obj_id': 1, 'title': 'testing python', 'secret': 'herp'},
])
def test_filters(self):
self.store_data()
f = lambda i: i['secret'] == 'herp'
results = self.engine.search_json('testing python', filters=[f])
self.assertEqual(self.sort_results(results), [
{'obj_id': 1, 'title': 'testing python', 'secret': 'herp'},
{'obj_id': 3, 'title': 'web testing python code', 'secret': 'herp'},
])
def test_simple(self):
self.engine.print_scores = True
self.engine.store('testing python')
self.engine.store('testing python code')
self.engine.store('web testing python code')
self.engine.store('unit tests with python')
results = self.engine.search('testing')
self.assertEqual(results, ['testing python', 'testing python code', 'web testing python code'])
results = self.engine.search('code')
self.assertEqual(results, ['testing python code', 'web testing python code'])
def test_correct_sorting(self):
strings = []
for i in range(26):
strings.append('aaaa%s' % chr(i + ord('a')))
if i > 0:
strings.append('aaa%sa' % chr(i + ord('a')))
random.shuffle(strings)
for s in strings:
self.engine.store(s)
results = self.engine.search('aaa')
self.assertEqual(results, sorted(strings))
results = self.engine.search('aaa', limit=30)
self.assertEqual(results, sorted(strings)[:30])
def test_removing_objects(self):
self.store_data()
self.engine.remove(1)
results = self.engine.search_json('testing')
self.assertEqual(self.sort_results(results), [
{'obj_id': 2, 'title': 'testing python code', 'secret': 'derp'},
{'obj_id': 3, 'title': 'web testing python code', 'secret': 'herp'},
])
self.store_data(1)
self.engine.remove(2)
results = self.engine.search_json('testing')
self.assertEqual(self.sort_results(results), [
{'obj_id': 1, 'title': 'testing python', 'secret': 'herp'},
{'obj_id': 3, 'title': 'web testing python code', 'secret': 'herp'},
])
def test_clean_phrase(self):
self.assertEqual(self.engine.clean_phrase('abc def ghi'), ['abc', 'def', 'ghi'])
self.assertEqual(self.engine.clean_phrase('a A tHe an a'), [])
self.assertEqual(self.engine.clean_phrase(''), [])
self.assertEqual(
self.engine.clean_phrase('The Best of times, the blurst of times'),
['best', 'times', 'blurst', 'times'])
def test_exists(self):
self.assertFalse(self.engine.exists('test'))
self.engine.store('test')
self.assertTrue(self.engine.exists('test'))
def test_removing_objects_in_depth(self):
# want to ensure that redis is cleaned up and does not become polluted
# with spurious keys when objects are removed
redis_client = self.engine.client
prefix = self.engine.prefix
initial_key_count = len(redis_client.keys())
# store the blog "testing python"
self.store_data(1)
# see how many keys we have in the db - check again in a bit
key_len = len(redis_client.keys())
self.store_data(2)
key_len2 = len(redis_client.keys())
self.assertTrue(key_len != key_len2)
self.engine.remove(2)
# back to the original amount of keys
self.assertEqual(len(redis_client.keys()), key_len)
self.engine.remove(1)
self.assertEqual(len(redis_client.keys()), initial_key_count)
def test_updating(self):
self.engine.store('id1', 'title one', 'd1', 't1')
self.engine.store('id2', 'title two', 'd2', 't2')
self.engine.store('id3', 'title three', 'd3', 't3')
results = self.engine.search('tit')
self.assertEqual(results, ['d1', 'd3', 'd2'])
# overwrite the data for id1
self.engine.store('id1', 'title one', 'D1', 't1')
results = self.engine.search('tit')
self.assertEqual(results, ['D1', 'd3', 'd2'])
# overwrite the data with a new title, will remove the title one refs
self.engine.store('id1', 'Herple One', 'done', 't1')
results = self.engine.search('tit')
self.assertEqual(results, ['d3', 'd2'])
results = self.engine.search('her')
self.assertEqual(results, ['done'])
self.engine.store('id1', 'title one', 'Done', 't1', False)
results = self.engine.search('tit')
self.assertEqual(results, ['Done', 'd3', 'd2'])
# this shows that when we don't clean up crap gets left around
results = self.engine.search('her')
self.assertEqual(results, ['Done'])