From 7afed3e897ffe071cb281d86e10562af397c4f86 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Sun, 20 Aug 2023 07:43:14 -0400 Subject: [PATCH] Attempting to add openai model. --- .vscode/settings.json | 7 ++++- apps/discover/models.py | 29 ++++++++++++++++++- config/requirements.txt | 2 ++ media/js/newsblur/reader/reader_popover.js | 8 +++-- .../newsblur/views/discover_feeds_popover.js | 11 +++++-- media/js/newsblur/views/story_titles_view.js | 2 ++ utils/ai_functions.py | 17 +++++++++++ 7 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 utils/ai_functions.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 28357ee35..3cb8333d1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,6 +36,11 @@ }, "nrf-connect.toolchain.path": "${nrf-connect.toolchain:1.9.1}", "C_Cpp.default.configurationProvider": "nrf-connect", - "editor.formatOnSave": false, + "editor.formatOnSave": true, "ansible.python.interpreterPath": "/opt/homebrew/bin/python3", + "editor.defaultFormatter": "ms-python.black-formatter", + "black-formatter.args": [ + "--line-length=110" + ], + "python.analysis.typeCheckingMode": "basic", } diff --git a/apps/discover/models.py b/apps/discover/models.py index beeb30826..47863fa6b 100644 --- a/apps/discover/models.py +++ b/apps/discover/models.py @@ -1,2 +1,29 @@ -from django.db import models +import openai +import mongoengine as mongo +from itertools import groupby +from apps.rss_feeds.models import Feed +from apps.reader.models import UserSubscription, UserSubscriptionFolders +from utils import json_functions as json +from utils.feed_functions import add_object_to_folder +from utils import log as logging +class MDiscoverFeed(mongo.Document): + feed_id = mongo.IntField() + related_feed_ids = mongo.ListField(mongo.IntField()) + + meta = { + 'collection': 'discover_feeds', + 'indexes': ['feed_id', 'related_feed_ids'], + 'allow_inheritance': False, + } + + def __str__(self): + feed = Feed.get_by_id(self.feed_id) + return "%s: related to %s sites" % (feed, len(self.related_feed_ids)) + + @classmethod + def fetch_related_feeds(feed_id, openai_model="gpt-3.5-turbo-16k", max_tokens=16000,): + feed = Feed.get_by_id(feed_id) + if not feed or not feed.feed_address: + return [] + diff --git a/config/requirements.txt b/config/requirements.txt index 769a0f810..0059cf292 100644 --- a/config/requirements.txt +++ b/config/requirements.txt @@ -74,6 +74,7 @@ nose-exclude==0.5.0 numpy==1.19.4 oauth2==1.9.0.post1 oauthlib==3.1.0 +openai~=0.27 packaging==20.9 paypalrestsdk==1.13.1 pbr==5.6.0 @@ -116,6 +117,7 @@ stevedore==3.3.0 stripe==2.55.1 subdomains==3.0.1 text-unidecode==1.3 +tiktoken~=0.4.0 toml==0.10.2 tweepy==3.9.0 urllib3==1.26.2 diff --git a/media/js/newsblur/reader/reader_popover.js b/media/js/newsblur/reader/reader_popover.js index 58be4e2b3..64bc63957 100644 --- a/media/js/newsblur/reader/reader_popover.js +++ b/media/js/newsblur/reader/reader_popover.js @@ -51,6 +51,12 @@ NEWSBLUR.ReaderPopover = Backbone.View.extend({ } // Need time to let animation begin so height registers as non-zero + this.check_height(); + + return this; + }, + + check_height: function () { _.defer(_.bind(function () { if ($(window).height() < (this.$popover.height() + 100)) { this.$popover.addClass('NB-popover-scroll'); @@ -58,8 +64,6 @@ NEWSBLUR.ReaderPopover = Backbone.View.extend({ this.$popover.width(this.options.width + 18); } }, this)); - - return this; }, close: function(e, hide_callback) { diff --git a/media/js/newsblur/views/discover_feeds_popover.js b/media/js/newsblur/views/discover_feeds_popover.js index 7df11b7d1..139ad9119 100644 --- a/media/js/newsblur/views/discover_feeds_popover.js +++ b/media/js/newsblur/views/discover_feeds_popover.js @@ -83,12 +83,17 @@ NEWSBLUR.DiscoverFeedsPopover = NEWSBLUR.ReaderPopover.extend({ this.$el.html($.make('div', [ $.make('div', { className: 'NB-popover-section' }, [ $.make('div', { className: 'NB-popover-section-title' }, 'Discover sites'), - $.make('div', { className: 'NB-discover-feed-badges' }, this.discover_feeds_model.map(function (discover_feed) { - return new NEWSBLUR.Views.FeedBadge({ model: discover_feed.get("feed") }); - })) + $.make('div', { className: 'NB-discover-feed-badges' }, _.flatten(this.discover_feeds_model.map(function (discover_feed) { + return [ + new NEWSBLUR.Views.FeedBadge({ model: discover_feed.get("feed") }), + new NEWSBLUR.Views.StoryTitlesView({ collection: discover_feed.get("stories"), on_dashboard: true }).render().el + ]; + }))) ]) ])); + this.check_height(); + return this; } diff --git a/media/js/newsblur/views/story_titles_view.js b/media/js/newsblur/views/story_titles_view.js index 19097516b..234d5269d 100644 --- a/media/js/newsblur/views/story_titles_view.js +++ b/media/js/newsblur/views/story_titles_view.js @@ -57,6 +57,8 @@ NEWSBLUR.Views.StoryTitlesView = Backbone.View.extend({ this.override_grid(); this.scroll_to_selected_story(null, options); + + return this; }, add: function(options) { diff --git a/utils/ai_functions.py b/utils/ai_functions.py new file mode 100644 index 000000000..8f9c354ea --- /dev/null +++ b/utils/ai_functions.py @@ -0,0 +1,17 @@ +import openai +import tiktoken +import logging +import re +from html import unescape + +from django.conf import settings + +def setup_openai_model(openai_model): + openai.api_key = settings.OPENAI_API_KEY + try: + encoding = tiktoken.encoding_for_model(openai_model) + except KeyError: + logging.debug(f"Could not find encoding for model {openai_model}, using cl100k_base") + encoding = tiktoken.get_encoding("cl100k_base") + + return encoding