From 3f7b80e4d1f7a4249a214798ff1d8fea3a6daeb8 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 21 Jan 2014 16:38:20 -0800 Subject: [PATCH] Updating tweepy to v2.2. --- vendor/tweepy/__init__.py | 4 +- vendor/tweepy/api.py | 84 ++++++++++++++++++----------- vendor/tweepy/auth.py | 23 +++----- vendor/tweepy/binder.py | 5 +- vendor/tweepy/cursor.py | 14 ++++- vendor/tweepy/models.py | 9 ++-- vendor/tweepy/streaming.py | 105 +++++++++++++++++++++++++++++++------ vendor/tweepy/utils.py | 49 ++--------------- 8 files changed, 175 insertions(+), 118 deletions(-) diff --git a/vendor/tweepy/__init__.py b/vendor/tweepy/__init__.py index 24529e7d4..05dbfc3f5 100755 --- a/vendor/tweepy/__init__.py +++ b/vendor/tweepy/__init__.py @@ -5,7 +5,7 @@ """ Tweepy Twitter API library """ -__version__ = '2.0' +__version__ = '2.2' __author__ = 'Joshua Roesslein' __license__ = 'MIT' @@ -13,7 +13,7 @@ from tweepy.models import Status, User, DirectMessage, Friendship, SavedSearch, from tweepy.error import TweepError from tweepy.api import API from tweepy.cache import Cache, MemoryCache, FileCache -from tweepy.auth import BasicAuthHandler, OAuthHandler +from tweepy.auth import OAuthHandler from tweepy.streaming import Stream, StreamListener from tweepy.cursor import Cursor diff --git a/vendor/tweepy/api.py b/vendor/tweepy/api.py index 74188093d..51a4bb6ae 100755 --- a/vendor/tweepy/api.py +++ b/vendor/tweepy/api.py @@ -57,14 +57,6 @@ class API(object): require_auth = True ) - """/statuses/:id/retweeted_by.format""" - retweeted_by = bind_api( - path = '/statuses/{id}/retweeted_by.json', - payload_type = 'status', payload_list = True, - allowed_param = ['id', 'count', 'page'], - require_auth = True - ) - """/related_results/show/:id.format""" related_results = bind_api( path = '/related_results/show/{id}.json', @@ -73,14 +65,6 @@ class API(object): require_auth = False ) - """/statuses/:id/retweeted_by/ids.format""" - retweeted_by_ids = bind_api( - path = '/statuses/{id}/retweeted_by/ids.json', - payload_type = 'ids', - allowed_param = ['id', 'count', 'page'], - require_auth = True - ) - """ statuses/retweets_of_me """ retweets_of_me = bind_api( path = '/statuses/retweets_of_me.json', @@ -105,6 +89,22 @@ class API(object): require_auth = True ) + """ statuses/update_with_media """ + def update_with_media(self, filename, *args, **kwargs): + headers, post_data = API._pack_image(filename, 3072, form_field='media[]') + kwargs.update({'headers': headers, 'post_data': post_data}) + + return bind_api( + path='/statuses/update_with_media.json', + method = 'POST', + payload_type='status', + allowed_param = [ + 'status', 'possibly_sensitive', 'in_reply_to_status_id', 'lat', 'long', + 'place_id', 'display_coordinates' + ], + require_auth=True + )(self, *args, **kwargs) + """ statuses/destroy """ destroy_status = bind_api( path = '/statuses/destroy/{id}.json', @@ -131,6 +131,12 @@ class API(object): require_auth = True ) + retweeters = bind_api( + path = '/statuses/retweeters/ids.json', + payload_type = 'ids', + allowed_param = ['id', 'cursor', 'stringify_ids'] + ) + """ users/show """ get_user = bind_api( path = '/users/show.json', @@ -310,7 +316,8 @@ class API(object): followers = bind_api( path = '/followers/list.json', payload_type = 'user', payload_list = True, - allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] + allowed_param = ['id', 'user_id', 'screen_name', 'cursor', 'count', + 'skip_status', 'include_user_entities'] ) """ account/verify_credentials """ @@ -376,6 +383,17 @@ class API(object): require_auth = True )(self, post_data=post_data, headers=headers) + """ account/update_profile_banner """ + def update_profile_banner(self, filename, *args, **kargs): + headers, post_data = API._pack_image(filename, 700, form_field="banner") + bind_api( + path = '/account/update_profile_banner.json', + method = 'POST', + allowed_param = ['width', 'height', 'offset_left', 'offset_right'], + require_auth = True + )(self, post_data=post_data, headers=headers) + + """ account/update_profile """ update_profile = bind_api( path = '/account/update_profile.json', @@ -485,16 +503,6 @@ class API(object): require_auth = True ) - """ help/test """ - def test(self): - try: - bind_api( - path = '/help/test.json', - )(self) - except TweepError: - return False - return True - create_list = bind_api( path = '/lists/create.json', method = 'POST', @@ -543,7 +551,7 @@ class API(object): list_timeline = bind_api( path = '/lists/statuses.json', payload_type = 'status', payload_list = True, - allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'since_id', 'max_id', 'count'] + allowed_param = ['owner_screen_name', 'slug', 'owner_id', 'list_id', 'since_id', 'max_id', 'count', 'include_rts'] ) get_list = bind_api( @@ -630,7 +638,7 @@ class API(object): search = bind_api( path = '/search/tweets.json', payload_type = 'search_results', - allowed_param = ['q', 'lang', 'locale', 'since_id', 'geocode', 'show_user', 'max_id', 'since', 'until', 'result_type'] + allowed_param = ['q', 'lang', 'locale', 'since_id', 'geocode', 'max_id', 'since', 'until', 'result_type', 'count', 'include_entities', 'from', 'to', 'source'] ) """ trends/daily """ @@ -675,9 +683,23 @@ class API(object): allowed_param = ['lat', 'long', 'name', 'contained_within'] ) + """ help/languages.json """ + supported_languages = bind_api( + path = '/help/languages.json', + payload_type = 'json', + require_auth = True + ) + + """ help/configuration """ + configuration = bind_api( + path = '/help/configuration.json', + payload_type = 'json', + require_auth = True + ) + """ Internal use only """ @staticmethod - def _pack_image(filename, max_size): + def _pack_image(filename, max_size, form_field="image"): """Pack image from file into multipart-formdata post body""" # image must be less than 700kb in size try: @@ -699,7 +721,7 @@ class API(object): BOUNDARY = 'Tw3ePy' body = [] body.append('--' + BOUNDARY) - body.append('Content-Disposition: form-data; name="image"; filename="%s"' % filename) + body.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (form_field, filename)) body.append('Content-Type: %s' % file_type) body.append('') body.append(fp.read()) diff --git a/vendor/tweepy/auth.py b/vendor/tweepy/auth.py index 27890aa43..29df5b216 100755 --- a/vendor/tweepy/auth.py +++ b/vendor/tweepy/auth.py @@ -21,26 +21,19 @@ class AuthHandler(object): raise NotImplementedError -class BasicAuthHandler(AuthHandler): - - def __init__(self, username, password): - self.username = username - self._b64up = base64.b64encode('%s:%s' % (username, password)) - - def apply_auth(self, url, method, headers, parameters): - headers['Authorization'] = 'Basic %s' % self._b64up - - def get_username(self): - return self.username - - class OAuthHandler(AuthHandler): """OAuth authentication handler""" OAUTH_HOST = 'api.twitter.com' OAUTH_ROOT = '/oauth/' - def __init__(self, consumer_key, consumer_secret, callback=None, secure=False): + def __init__(self, consumer_key, consumer_secret, callback=None, secure=True): + if type(consumer_key) == unicode: + consumer_key = bytes(consumer_key) + + if type(consumer_secret) == unicode: + consumer_secret = bytes(consumer_secret) + self._consumer = oauth.OAuthConsumer(consumer_key, consumer_secret) self._sigmethod = oauth.OAuthSignatureMethod_HMAC_SHA1() self.request_token = None @@ -49,7 +42,7 @@ class OAuthHandler(AuthHandler): self.username = None self.secure = secure - def _get_oauth_url(self, endpoint, secure=False): + def _get_oauth_url(self, endpoint, secure=True): if self.secure or secure: prefix = 'https://' else: diff --git a/vendor/tweepy/binder.py b/vendor/tweepy/binder.py index 079721506..29eacaedf 100755 --- a/vendor/tweepy/binder.py +++ b/vendor/tweepy/binder.py @@ -104,6 +104,8 @@ def bind_api(**config): self.path = self.path.replace(variable, value) def execute(self): + self.api.cached_result = False + # Build the request URL url = self.api_root + self.path if len(self.parameters): @@ -123,6 +125,7 @@ def bind_api(**config): else: if isinstance(cache_result, Model): cache_result._api = self.api + self.api.cached_result = True return cache_result # Continue attempting request until successful @@ -165,7 +168,7 @@ def bind_api(**config): # If an error was returned, throw an exception self.api.last_response = resp - if resp.status != 200: + if resp.status and not 200 <= resp.status < 300: try: error_msg = self.api.parser.parse_error(resp.read()) except Exception: diff --git a/vendor/tweepy/cursor.py b/vendor/tweepy/cursor.py index edda0ba62..4c06f17a9 100755 --- a/vendor/tweepy/cursor.py +++ b/vendor/tweepy/cursor.py @@ -53,8 +53,9 @@ class CursorIterator(BaseIterator): def __init__(self, method, args, kargs): BaseIterator.__init__(self, method, args, kargs) - self.next_cursor = -1 - self.prev_cursor = 0 + start_cursor = kargs.pop('cursor', None) + self.next_cursor = start_cursor or -1 + self.prev_cursor = start_cursor or 0 self.count = 0 def next(self): @@ -84,9 +85,13 @@ class IdIterator(BaseIterator): BaseIterator.__init__(self, method, args, kargs) self.max_id = kargs.get('max_id') self.since_id = kargs.get('since_id') + self.count = 0 def next(self): """Fetch a set of items with IDs less than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + # max_id is inclusive so decrement by one # to avoid requesting duplicate items. max_id = self.since_id - 1 if self.max_id else None @@ -95,16 +100,21 @@ class IdIterator(BaseIterator): raise StopIteration self.max_id = data.max_id self.since_id = data.since_id + self.count += 1 return data def prev(self): """Fetch a set of items with IDs greater than current set.""" + if self.limit and self.limit == self.count: + raise StopIteration + since_id = self.max_id data = self.method(since_id = since_id, *self.args, **self.kargs) if len(data) == 0: raise StopIteration self.max_id = data.max_id self.since_id = data.since_id + self.count += 1 return data class PageIterator(BaseIterator): diff --git a/vendor/tweepy/models.py b/vendor/tweepy/models.py index 34427900e..69d344d7b 100755 --- a/vendor/tweepy/models.py +++ b/vendor/tweepy/models.py @@ -3,8 +3,7 @@ # See LICENSE for details. from tweepy.error import TweepError -from tweepy.utils import parse_datetime, parse_html_value, parse_a_href, \ - parse_search_datetime, unescape_html +from tweepy.utils import parse_datetime, parse_html_value, parse_a_href class ResultSet(list): @@ -67,7 +66,7 @@ class Status(Model): status = cls(api) for k, v in json.items(): if k == 'user': - user_model = getattr(api.parser.model_factory, 'user') + user_model = getattr(api.parser.model_factory, 'user') if api else User user = user_model.parse(api, v) setattr(status, 'author', user) setattr(status, 'user', user) # DEPRECIATED @@ -160,7 +159,7 @@ class User(Model): return self._api.lists_subscriptions(user=self.screen_name, *args, **kargs) def lists(self, *args, **kargs): - return self._api.lists(user=self.screen_name, *args, **kargs) + return self._api.lists_all(user=self.screen_name, *args, **kargs) def followers_ids(self, *args, **kargs): return self._api.followers_ids(user_id=self.id, *args, **kargs) @@ -238,6 +237,8 @@ class SearchResults(ResultSet): results.refresh_url = metadata.get('refresh_url') results.completed_in = metadata.get('completed_in') results.query = metadata.get('query') + results.count = metadata.get('count') + results.next_results = metadata.get('next_results') for status in json['statuses']: results.append(Status.parse(api, status)) diff --git a/vendor/tweepy/streaming.py b/vendor/tweepy/streaming.py index f6d37f450..b17b4f96c 100755 --- a/vendor/tweepy/streaming.py +++ b/vendor/tweepy/streaming.py @@ -2,10 +2,12 @@ # Copyright 2009-2010 Joshua Roesslein # See LICENSE for details. +import logging import httplib from socket import timeout from threading import Thread from time import sleep +import ssl from tweepy.models import Status from tweepy.api import API @@ -31,33 +33,59 @@ class StreamListener(object): """ pass - def on_data(self, data): + def on_data(self, raw_data): """Called when raw data is received from connection. Override this method if you wish to manually handle the stream data. Return False to stop stream and close connection. """ + data = json.loads(raw_data) if 'in_reply_to_status_id' in data: - status = Status.parse(self.api, json.loads(data)) + status = Status.parse(self.api, data) if self.on_status(status) is False: return False elif 'delete' in data: - delete = json.loads(data)['delete']['status'] + delete = data['delete']['status'] if self.on_delete(delete['id'], delete['user_id']) is False: return False - elif 'limit' in data: - if self.on_limit(json.loads(data)['limit']['track']) is False: + elif 'event' in data: + status = Status.parse(self.api, data) + if self.on_event(status) is False: return False + elif 'direct_message' in data: + status = Status.parse(self.api, data) + if self.on_direct_message(status) is False: + return False + elif 'limit' in data: + if self.on_limit(data['limit']['track']) is False: + return False + elif 'disconnect' in data: + if self.on_disconnect(data['disconnect']) is False: + return False + else: + logging.error("Unknown message type: " + str(raw_data)) def on_status(self, status): """Called when a new status arrives""" return + def on_exception(self, exception): + """Called when an unhandled exception occurs.""" + return + def on_delete(self, status_id, user_id): """Called when a delete notice arrives for a status""" return + def on_event(self, status): + """Called when a new event arrives""" + return + + def on_direct_message(self, status): + """Called when a new direct message arrives""" + return + def on_limit(self, track): """Called when a limitation notice arrvies""" return @@ -70,6 +98,14 @@ class StreamListener(object): """Called when stream connection times out""" return + def on_disconnect(self, notice): + """Called when twitter sends a disconnect notice + + Disconnect codes are listed here: + https://dev.twitter.com/docs/streaming-apis/messages#Disconnect_messages_disconnect + """ + return + class Stream(object): @@ -81,8 +117,12 @@ class Stream(object): self.running = False self.timeout = options.get("timeout", 300.0) self.retry_count = options.get("retry_count") - self.retry_time = options.get("retry_time", 10.0) - self.snooze_time = options.get("snooze_time", 5.0) + # values according to https://dev.twitter.com/docs/streaming-apis/connecting#Reconnecting + self.retry_time_start = options.get("retry_time", 5.0) + self.retry_420_start = options.get("retry_420", 60.0) + self.retry_time_cap = options.get("retry_time_cap", 320.0) + self.snooze_time_step = options.get("snooze_time", 0.25) + self.snooze_time_cap = options.get("snooze_time_cap", 16) self.buffer_size = options.get("buffer_size", 1500) if options.get("secure", True): self.scheme = "https" @@ -93,6 +133,8 @@ class Stream(object): self.headers = options.get("headers") or {} self.parameters = None self.body = None + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step def _run(self): # Authenticate @@ -108,30 +150,41 @@ class Stream(object): break try: if self.scheme == "http": - conn = httplib.HTTPConnection(self.host) + conn = httplib.HTTPConnection(self.host, timeout=self.timeout) else: - conn = httplib.HTTPSConnection(self.host) + conn = httplib.HTTPSConnection(self.host, timeout=self.timeout) self.auth.apply_auth(url, 'POST', self.headers, self.parameters) conn.connect() - conn.sock.settimeout(self.timeout) conn.request('POST', self.url, self.body, headers=self.headers) resp = conn.getresponse() if resp.status != 200: if self.listener.on_error(resp.status) is False: break error_counter += 1 + if resp.status == 420: + self.retry_time = max(self.retry_420_start, self.retry_time) sleep(self.retry_time) + self.retry_time = min(self.retry_time * 2, self.retry_time_cap) else: error_counter = 0 + self.retry_time = self.retry_time_start + self.snooze_time = self.snooze_time_step self.listener.on_connect() self._read_loop(resp) - except timeout: + except (timeout, ssl.SSLError), exc: + # If it's not time out treat it like any other exception + if isinstance(exc, ssl.SSLError) and not (exc.args and 'timed out' in str(exc.args[0])): + exception = exc + break + if self.listener.on_timeout() == False: break if self.running is False: break conn.close() sleep(self.snooze_time) + self.snooze_time = min(self.snooze_time + self.snooze_time_step, + self.snooze_time_cap) except Exception, exception: # any other exception is fatal, so kill loop break @@ -142,6 +195,8 @@ class Stream(object): conn.close() if exception: + # call a handler first so that the exception can be logged. + self.listener.on_exception(exception) raise def _data(self, data): @@ -184,12 +239,26 @@ class Stream(object): """ Called when the response has been closed by Twitter """ pass - def userstream(self, count=None, async=False, secure=True): + def userstream(self, stall_warnings=False, _with=None, replies=None, + track=None, locations=None, async=False, encoding='utf8'): self.parameters = {'delimited': 'length'} if self.running: raise TweepError('Stream object already connected!') - self.url = '/2/user.json?delimited=length' + self.url = '/%s/user.json?delimited=length' % STREAM_VERSION self.host='userstream.twitter.com' + if stall_warnings: + self.parameters['stall_warnings'] = stall_warnings + if _with: + self.parameters['with'] = _with + if replies: + self.parameters['replies'] = replies + if locations and len(locations) > 0: + assert len(locations) % 4 == 0 + self.parameters['locations'] = ','.join(['%.2f' % l for l in locations]) + if track: + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) + self.body = urlencode_noplus(self.parameters) self._start(async) def firehose(self, count=None, async=False): @@ -217,17 +286,19 @@ class Stream(object): self.url += '&count=%s' % count self._start(async) - def filter(self, follow=None, track=None, async=False, locations=None, - count = None, stall_warnings=False, languages=None): + def filter(self, follow=None, track=None, async=False, locations=None, + count=None, stall_warnings=False, languages=None, encoding='utf8'): self.parameters = {} self.headers['Content-type'] = "application/x-www-form-urlencoded" if self.running: raise TweepError('Stream object already connected!') self.url = '/%s/statuses/filter.json?delimited=length' % STREAM_VERSION if follow: - self.parameters['follow'] = ','.join(map(str, follow)) + encoded_follow = [s.encode(encoding) for s in follow] + self.parameters['follow'] = ','.join(encoded_follow) if track: - self.parameters['track'] = ','.join(map(str, track)) + encoded_track = [s.encode(encoding) for s in track] + self.parameters['track'] = ','.join(encoded_track) if locations and len(locations) > 0: assert len(locations) % 4 == 0 self.parameters['locations'] = ','.join(['%.2f' % l for l in locations]) diff --git a/vendor/tweepy/utils.py b/vendor/tweepy/utils.py index 52c6c79b9..7c2d4987a 100755 --- a/vendor/tweepy/utils.py +++ b/vendor/tweepy/utils.py @@ -8,18 +8,11 @@ import htmlentitydefs import re import locale from urllib import quote +from email.utils import parsedate def parse_datetime(string): - # Set locale for date parsing - locale.setlocale(locale.LC_TIME, 'C') - - # We must parse datetime this way to work in python 2.4 - date = datetime(*(time.strptime(string, '%a %b %d %H:%M:%S +0000 %Y')[0:6])) - - # Reset locale back to the default setting - locale.setlocale(locale.LC_TIME, '') - return date + return datetime(*(parsedate(string)[:6])) def parse_html_value(html): @@ -34,41 +27,6 @@ def parse_a_href(atag): return atag[start:end] -def parse_search_datetime(string): - # Set locale for date parsing - locale.setlocale(locale.LC_TIME, 'C') - - # We must parse datetime this way to work in python 2.4 - date = datetime(*(time.strptime(string, '%a, %d %b %Y %H:%M:%S +0000')[0:6])) - - # Reset locale back to the default setting - locale.setlocale(locale.LC_TIME, '') - return date - - -def unescape_html(text): - """Created by Fredrik Lundh (http://effbot.org/zone/re-sub.htm#unescape-html)""" - def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) - except KeyError: - pass - return text # leave as is - return re.sub("&#?\w+;", fixup, text) - - def convert_to_utf8_str(arg): # written by Michael Norton (http://docondev.blogspot.com/) if isinstance(arg, unicode): @@ -98,6 +56,5 @@ def list_to_csv(item_list): return ','.join([str(i) for i in item_list]) def urlencode_noplus(query): - return '&'.join(['%s=%s' % (quote(str(k)), quote(str(v))) \ + return '&'.join(['%s=%s' % (quote(str(k), ''), quote(str(v), '')) \ for k, v in query.iteritems()]) -