public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-04-13 09:42:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Splitting redis query logging into three dbs: user, story, and session.

Browse source
This commit is contained in:
Samuel Clay 2021-08-03 16:05:48 -04:00
parent 8348da9c59
commit abfacef9c9
3 changed files with 37 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
apps/profile/middleware.py
View file

@ -146,8 +146,14 @@ class SQLLogToConsoleMiddleware:
for query in queries: for query in queries:
if query.get('mongo'): if query.get('mongo'):
query['sql'] = "~FM%s: %s" % (query['mongo']['collection'], query['mongo']['query']) query['sql'] = "~FM%s: %s" % (query['mongo']['collection'], query['mongo']['query'])
elif query.get('redis'): elif query.get('db_redis'):
query['sql'] = "~FC%s" % (query['redis']['query']) query['sql'] = "~FC%s" % (query['db_redis']['query'])
elif query.get('redis_user'):
query['sql'] = "~FC%s" % (query['redis_user']['query'])
elif query.get('redis_story'):
query['sql'] = "~FC%s" % (query['redis_story']['query'])
elif query.get('redis_session'):
query['sql'] = "~FC%s" % (query['redis_session']['query'])
else: else:
query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql']) query['sql'] = re.sub(r'SELECT (.*?) FROM', 'SELECT * FROM', query['sql'])
query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql']) query['sql'] = re.sub(r'SELECT', '~FYSELECT', query['sql'])
@ -164,9 +170,15 @@ class SQLLogToConsoleMiddleware:
times_elapsed = { times_elapsed = {
'sql': sum([float(q['time']) 'sql': sum([float(q['time'])
for q in queries if not q.get('mongo') and for q in queries if not q.get('mongo') and
not q.get('redis')]), not q.get('redis_user') and
not q.get('redis_story') and
not q.get('redis_session') and
not q.get('db_redis')]),
'mongo': sum([float(q['time']) for q in queries if q.get('mongo')]), 'mongo': sum([float(q['time']) for q in queries if q.get('mongo')]),
'redis': sum([float(q['time']) for q in queries if q.get('redis')]), 'db_redis': sum([float(q['time']) for q in queries if q.get('db_redis')]),
'redis_user': sum([float(q['time']) for q in queries if q.get('redis_user')]),
'redis_story': sum([float(q['time']) for q in queries if q.get('redis_story')]),
'redis_session': sum([float(q['time']) for q in queries if q.get('redis_session')]),
} }
setattr(request, 'sql_times_elapsed', times_elapsed) setattr(request, 'sql_times_elapsed', times_elapsed)
else: else:

7
utils/redis_raw_log_middleware.py
View file

@ -50,7 +50,7 @@ class RedisDumpMiddleware(object):
if not getattr(connection, 'queriesx', False): if not getattr(connection, 'queriesx', False):
connection.queriesx = [] connection.queriesx = []
connection.queriesx.append({ connection.queriesx.append({
'redis': message, message['redis_server_name']: message,
'time': '%.6f' % duration, 'time': '%.6f' % duration,
}) })
return result return result
@ -58,13 +58,16 @@ class RedisDumpMiddleware(object):
def process_message(self, *args, **kwargs): def process_message(self, *args, **kwargs):
query = [] query = []
redis_server_name = None
for a, arg in enumerate(args): for a, arg in enumerate(args):
if isinstance(arg, Connection): if isinstance(arg, Connection):
redis_connection = arg
redis_server_name = redis_connection.host
continue continue
if len(str(arg)) > 100: if len(str(arg)) > 100:
arg = "[%s bytes]" % len(str(arg)) arg = "[%s bytes]" % len(str(arg))
query.append(str(arg).replace('\n', '')) query.append(str(arg).replace('\n', ''))
return { 'query': ' '.join(query) } return { 'query': ' '.join(query), 'redis_server_name': redis_server_name }
def __call__(self, request): def __call__(self, request):
response = None response = None

19
utils/request_introspection_middleware.py
View file

@ -21,14 +21,27 @@ class DumpRequestMiddleware:
def process_response(self, request, response): def process_response(self, request, response):
if hasattr(request, 'sql_times_elapsed'): if hasattr(request, 'sql_times_elapsed'):
logging.debug(" ---> %s~SN~FCDB times: ~FYsql: %s%.4f~SNs ~SN~FMmongo: %s%.5f~SNs ~SN~FCredis: %s%.6f~SNs" % ( if request.sql_times_elapsed.get('redis_user'):
redis_log = "user:%s%.6f~SNs story:%s%.6f~SNs session:%s%.6f~SNs" % (
self.color_db(request.sql_times_elapsed['redis_user'], '~FC'),
request.sql_times_elapsed['redis_user'],
self.color_db(request.sql_times_elapsed['redis_story'], '~FC'),
request.sql_times_elapsed['redis_story'],
self.color_db(request.sql_times_elapsed['redis_session'], '~FC'),
request.sql_times_elapsed['redis_session'],
)
else:
redis_log = "%s%.6f~SNs" % (
self.color_db(request.sql_times_elapsed['db_redis'], '~FC'),
request.sql_times_elapsed['db_redis'],
)
logging.debug(" ---> %s~SN~FCDB times: ~FYsql: %s%.4f~SNs ~SN~FMmongo: %s%.5f~SNs ~SN~FCredis: %s" % (
self.elapsed_time(request), self.elapsed_time(request),
self.color_db(request.sql_times_elapsed['sql'], '~FY'), self.color_db(request.sql_times_elapsed['sql'], '~FY'),
request.sql_times_elapsed['sql'], request.sql_times_elapsed['sql'],
self.color_db(request.sql_times_elapsed['mongo'], '~FM'), self.color_db(request.sql_times_elapsed['mongo'], '~FM'),
request.sql_times_elapsed['mongo'], request.sql_times_elapsed['mongo'],
self.color_db(request.sql_times_elapsed['redis'], '~FC'), redis_log
request.sql_times_elapsed['redis'],
)) ))
return response return response