mirror of
https://github.com/samuelclay/NewsBlur.git
synced 2025-08-21 05:45:13 +00:00
18 lines
461 B
Python
18 lines
461 B
Python
|
import openai
|
||
|
|
import tiktoken
|
||
|
|
import logging
|
||
|
|
import re
|
||
|
|
from html import unescape
|
||
|
|
|
||
|
|
from django.conf import settings
|
||
|
|
|
||
|
|
def setup_openai_model(openai_model):
|
||
|
|
openai.api_key = settings.OPENAI_API_KEY
|
||
|
|
try:
|
||
|
|
encoding = tiktoken.encoding_for_model(openai_model)
|
||
|
|
except KeyError:
|
||
|
|
logging.debug(f"Could not find encoding for model {openai_model}, using cl100k_base")
|
||
|
|
encoding = tiktoken.get_encoding("cl100k_base")
|
||
|
|
|
||
|
|
return encoding
|