public-mirrors/NewsBlur
1
0
Fork 0
mirror of https://github.com/samuelclay/NewsBlur.git synced 2025-04-13 09:42:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reducing dbs.

Browse source
This commit is contained in:
Samuel Clay 2023-06-08 07:51:39 -04:00
parent 2167805856
commit cd2a32bee3
3 changed files with 99 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

174
Makefile
View file

@ -1,108 +1,108 @@
SHELL := /bin/bash SHELL := /bin/bash
CURRENT_UID := $(shell id -u) CURRENT_UID := $(shell id -u)
CURRENT_GID := $(shell id -g) CURRENT_GID := $(shell id -g)
newsblur := $(shell docker ps -qf "name=newsblur_web") newsblur := $(shell gtimeout 2s docker ps -qf "name=newsblur_web")
.PHONY: node .PHONY: node
nb: pull bounce migrate bootstrap collectstatic nb: pull bounce migrate bootstrap collectstatic
metrics: metrics:
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose -f docker-compose.yml -f docker-compose.metrics.yml up -d RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose -f docker-compose.yml -f docker-compose.metrics.yml up -d
collectstatic: collectstatic:
- rm -fr static rm -fr static
- docker pull newsblur/newsblur_deploy docker pull newsblur/newsblur_deploy
- docker run --rm -v $(shell pwd):/srv/newsblur newsblur/newsblur_deploy docker run --rm -v $(shell pwd):/srv/newsblur newsblur/newsblur_deploy
#creates newsblur, builds new images, and creates/refreshes SSL keys #creates newsblur, builds new images, and creates/refreshes SSL keys
bounce: bounce:
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose down RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose down
- [[ -d config/certificates ]] && echo "keys exist" || make keys [[ -d config/certificates ]] && echo "keys exist" || make keys
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose up -d --build --remove-orphans RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose up -d --build --remove-orphans
bootstrap: bootstrap:
- docker exec newsblur_web ./manage.py loaddata config/fixtures/bootstrap.json docker exec newsblur_web ./manage.py loaddata config/fixtures/bootstrap.json
nbup: nbup:
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose up -d --build --remove-orphans RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose up -d --build --remove-orphans
coffee: coffee:
- coffee -c -w **/*.coffee coffee -c -w **/*.coffee
migrations: migrations:
- docker exec -it newsblur_web ./manage.py makemigrations docker exec -it newsblur_web ./manage.py makemigrations
makemigration: migrations makemigration: migrations
datamigration: datamigration:
- docker exec -it newsblur_web ./manage.py makemigrations --empty $(app) docker exec -it newsblur_web ./manage.py makemigrations --empty $(app)
migration: migrations migration: migrations
migrate: migrate:
- docker exec -it newsblur_web ./manage.py migrate docker exec -it newsblur_web ./manage.py migrate
shell: shell:
- docker exec -it newsblur_web ./manage.py shell_plus docker exec -it newsblur_web ./manage.py shell_plus
bash: bash:
- docker exec -it newsblur_web bash docker exec -it newsblur_web bash
# allows user to exec into newsblur_web and use pdb. # allows user to exec into newsblur_web and use pdb.
debug: debug:
- docker attach ${newsblur} docker attach ${newsblur}
log: log:
- RUNWITHMAKEBUILD=True docker compose logs -f --tail 20 newsblur_web newsblur_node RUNWITHMAKEBUILD=True docker compose logs -f --tail 20 newsblur_web newsblur_node
logweb: log logweb: log
logcelery: logcelery:
- RUNWITHMAKEBUILD=True docker compose logs -f --tail 20 task_celery RUNWITHMAKEBUILD=True docker compose logs -f --tail 20 task_celery
logtask: logcelery logtask: logcelery
logmongo: logmongo:
- RUNWITHMAKEBUILD=True docker compose logs -f db_mongo RUNWITHMAKEBUILD=True docker compose logs -f db_mongo
alllogs: alllogs:
- RUNWITHMAKEBUILD=True docker compose logs -f --tail 20 RUNWITHMAKEBUILD=True docker compose logs -f --tail 20
logall: alllogs logall: alllogs
mongo: mongo:
- docker exec -it db_mongo mongo --port 29019 docker exec -it db_mongo mongo --port 29019
redis: redis:
- docker exec -it db_redis redis-cli -p 6579 docker exec -it db_redis redis-cli -p 6579
postgres: postgres:
- docker exec -it db_postgres psql -U newsblur docker exec -it db_postgres psql -U newsblur
stripe: stripe:
- stripe listen --forward-to localhost/zebra/webhooks/v2/ stripe listen --forward-to localhost/zebra/webhooks/v2/
down: down:
- RUNWITHMAKEBUILD=True docker compose -f docker-compose.yml -f docker-compose.metrics.yml down RUNWITHMAKEBUILD=True docker compose -f docker-compose.yml -f docker-compose.metrics.yml down
nbdown: down nbdown: down
jekyll: jekyll:
- cd blog && bundle exec jekyll serve cd blog && bundle exec jekyll serve
jekyll_drafts: jekyll_drafts:
- cd blog && bundle exec jekyll serve --drafts cd blog && bundle exec jekyll serve --drafts
# runs tests # runs tests
test: test:
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} TEST=True docker compose -f docker-compose.yml up -d newsblur_web RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} TEST=True docker compose -f docker-compose.yml up -d newsblur_web
- RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose exec newsblur_web bash -c "NOSE_EXCLUDE_DIRS=./vendor DJANGO_SETTINGS_MODULE=newsblur_web.test_settings python3 manage.py test -v 3 --failfast" RUNWITHMAKEBUILD=True CURRENT_UID=${CURRENT_UID} CURRENT_GID=${CURRENT_GID} docker compose exec newsblur_web bash -c "NOSE_EXCLUDE_DIRS=./vendor DJANGO_SETTINGS_MODULE=newsblur_web.test_settings python3 manage.py test -v 3 --failfast"
keys: keys:
- mkdir config/certificates mkdir config/certificates
- openssl dhparam -out config/certificates/dhparam-2048.pem 2048 openssl dhparam -out config/certificates/dhparam-2048.pem 2048
- openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout config/certificates/RootCA.key -out config/certificates/RootCA.pem -subj "/C=US/CN=Example-Root-CA" openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout config/certificates/RootCA.key -out config/certificates/RootCA.pem -subj "/C=US/CN=Example-Root-CA"
- openssl x509 -outform pem -in config/certificates/RootCA.pem -out config/certificates/RootCA.crt openssl x509 -outform pem -in config/certificates/RootCA.pem -out config/certificates/RootCA.crt
- openssl req -new -nodes -newkey rsa:2048 -keyout config/certificates/localhost.key -out config/certificates/localhost.csr -subj "/C=US/ST=YourState/L=YourCity/O=Example-Certificates/CN=localhost" openssl req -new -nodes -newkey rsa:2048 -keyout config/certificates/localhost.key -out config/certificates/localhost.csr -subj "/C=US/ST=YourState/L=YourCity/O=Example-Certificates/CN=localhost"
- openssl x509 -req -sha256 -days 1024 -in config/certificates/localhost.csr -CA config/certificates/RootCA.pem -CAkey config/certificates/RootCA.key -CAcreateserial -out config/certificates/localhost.crt openssl x509 -req -sha256 -days 1024 -in config/certificates/localhost.csr -CA config/certificates/RootCA.pem -CAkey config/certificates/RootCA.key -CAcreateserial -out config/certificates/localhost.crt
- cat config/certificates/localhost.crt config/certificates/localhost.key > config/certificates/localhost.pem cat config/certificates/localhost.crt config/certificates/localhost.key > config/certificates/localhost.pem
- sudo /usr/bin/security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./config/certificates/RootCA.crt sudo /usr/bin/security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./config/certificates/RootCA.crt
# Doesn't work yet # Doesn't work yet
mkcert: mkcert:
- mkdir config/mkcert mkdir config/mkcert
- docker run -v $(shell pwd)/config/mkcert:/root/.local/share/mkcert brunopadz/mkcert-docker:latest \ docker run -v $(shell pwd)/config/mkcert:/root/.local/share/mkcert brunopadz/mkcert-docker:latest \
/bin/sh -c "mkcert -install && \ /bin/sh -c "mkcert -install && \
mkcert -cert-file /root/.local/share/mkcert/mkcert.pem \ mkcert -cert-file /root/.local/share/mkcert/mkcert.pem \
-key-file /root/.local/share/mkcert/mkcert.key localhost" -key-file /root/.local/share/mkcert/mkcert.key localhost"
- cat config/mkcert/rootCA.pem config/mkcert/rootCA-key.pem > config/certificates/localhost.pem cat config/mkcert/rootCA.pem config/mkcert/rootCA-key.pem > config/certificates/localhost.pem
- sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./config/mkcert/rootCA.pem sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ./config/mkcert/rootCA.pem
# Digital Ocean / Terraform # Digital Ocean / Terraform
list: list:
- doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` compute droplet list doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` compute droplet list
sizes: sizes:
- doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` compute size list doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` compute size list
size: sizes size: sizes
ratelimit: ratelimit:
- doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` account ratelimit doctl -t `cat /srv/secrets-newsblur/keys/digital_ocean.token` account ratelimit
ansible-deps: ansible-deps:
ansible-galaxy install -p roles -r ansible/roles/requirements.yml --roles-path ansible/roles ansible-galaxy install -p roles -r ansible/roles/requirements.yml --roles-path ansible/roles
tfrefresh: tfrefresh:
@ -112,95 +112,95 @@ plan:
apply: apply:
terraform -chdir=terraform apply -refresh=false -parallelism=15 terraform -chdir=terraform apply -refresh=false -parallelism=15
inventory: inventory:
- ./ansible/utils/generate_inventory.py ./ansible/utils/generate_inventory.py
oldinventory: oldinventory:
- OLD=1 ./ansible/utils/generate_inventory.py OLD=1 ./ansible/utils/generate_inventory.py
# Docker # Docker
pull: pull:
- docker pull newsblur/newsblur_python3 docker pull newsblur/newsblur_python3
- docker pull newsblur/newsblur_node docker pull newsblur/newsblur_node
- docker pull newsblur/newsblur_monitor docker pull newsblur/newsblur_monitor
local_build_web: local_build_web:
# - docker buildx build --load . --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3 # docker buildx build --load . --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
- docker build . --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3 docker build . --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
build_web: build_web:
- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3 docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
build_node: build_node:
- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node
build_monitor: build_monitor:
- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor
build_deploy: build_deploy:
- docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy docker buildx build . --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy
build: build_web build_node build_monitor build_deploy build: build_web build_node build_monitor build_deploy
push_web: push_web:
- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3 docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_base_image.Dockerfile --tag=newsblur/newsblur_python3
push_node: push_node:
- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/node/Dockerfile --tag=newsblur/newsblur_node
push_monitor: push_monitor:
- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/monitor/Dockerfile --tag=newsblur/newsblur_monitor
push_deploy: push_deploy:
- docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy docker buildx build . --push --platform linux/amd64,linux/arm64 --file=docker/newsblur_deploy.Dockerfile --tag=newsblur/newsblur_deploy
push_images: push_web push_node push_monitor push_deploy push_images: push_web push_node push_monitor push_deploy
push: push_images push: push_images
# Tasks # Tasks
deploy_web: deploy_web:
- ansible-playbook ansible/deploy.yml -l app ansible-playbook ansible/deploy.yml -l app
deploy: deploy_web deploy: deploy_web
app: deploy_web app: deploy_web
web: deploy_web web: deploy_web
deploy_static: deploy_static:
- ansible-playbook ansible/deploy.yml -l app --tags static ansible-playbook ansible/deploy.yml -l app --tags static
static: deploy_static static: deploy_static
deploy_node: deploy_node:
- ansible-playbook ansible/deploy.yml -l node ansible-playbook ansible/deploy.yml -l node
node: deploy_node node: deploy_node
deploy_task: deploy_task:
- ansible-playbook ansible/deploy.yml -l task ansible-playbook ansible/deploy.yml -l task
task: deploy_task task: deploy_task
celery: deploy_task celery: deploy_task
deploy_www: deploy_www:
- ansible-playbook ansible/deploy.yml -l haproxy ansible-playbook ansible/deploy.yml -l haproxy
www: deploy_www www: deploy_www
deploy_work: deploy_work:
- ansible-playbook ansible/deploy.yml -l work ansible-playbook ansible/deploy.yml -l work
work: deploy_work work: deploy_work
deploy_monitor: deploy_monitor:
- ansible-playbook ansible/deploy.yml -l db ansible-playbook ansible/deploy.yml -l db
monitor: deploy_monitor monitor: deploy_monitor
deploy_staging: deploy_staging:
- ansible-playbook ansible/deploy.yml -l staging ansible-playbook ansible/deploy.yml -l staging
staging: deploy_staging staging: deploy_staging
celery_stop: celery_stop:
- ansible-playbook ansible/deploy.yml -l task --tags stop ansible-playbook ansible/deploy.yml -l task --tags stop
sentry: sentry:
- ansible-playbook ansible/setup.yml -l sentry -t sentry ansible-playbook ansible/setup.yml -l sentry -t sentry
maintenance_on: maintenance_on:
- ansible-playbook ansible/deploy.yml -l web --tags maintenance_on ansible-playbook ansible/deploy.yml -l web --tags maintenance_on
maintenance_off: maintenance_off:
- ansible-playbook ansible/deploy.yml -l web --tags maintenance_off ansible-playbook ansible/deploy.yml -l web --tags maintenance_off
# Provision # Provision
firewall: firewall:
- ansible-playbook ansible/all.yml -l db --tags firewall ansible-playbook ansible/all.yml -l db --tags firewall
oldfirewall: oldfirewall:
- ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml -l db --tags firewall ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml -l db --tags firewall
repairmongo: repairmongo:
- sudo docker run -v "/srv/newsblur/docker/volumes/db_mongo:/data/db" mongo:4.0 mongod --repair --dbpath /data/db sudo docker run -v "/srv/newsblur/docker/volumes/db_mongo:/data/db" mongo:4.0 mongod --repair --dbpath /data/db
mongodump: mongodump:
- docker exec -it db_mongo mongodump --port 29019 -d newsblur -o /data/mongodump docker exec -it db_mongo mongodump --port 29019 -d newsblur -o /data/mongodump
- cp -fr docker/volumes/db_mongo/mongodump docker/volumes/mongodump cp -fr docker/volumes/db_mongo/mongodump docker/volumes/mongodump
# - docker exec -it db_mongo cp -fr /data/db/mongodump /data/mongodump # docker exec -it db_mongo cp -fr /data/db/mongodump /data/mongodump
# - docker exec -it db_mongo rm -fr /data/db/ # docker exec -it db_mongo rm -fr /data/db/
mongorestore: mongorestore:
- cp -fr docker/volumes/mongodump docker/volumes/db_mongo/ cp -fr docker/volumes/mongodump docker/volumes/db_mongo/
- docker exec -it db_mongo mongorestore --port 29019 -d newsblur /data/db/mongodump/newsblur docker exec -it db_mongo mongorestore --port 29019 -d newsblur /data/db/mongodump/newsblur
index_feeds: index_feeds:
- docker exec -it newsblur_web ./manage.py index_feeds docker exec -it newsblur_web ./manage.py index_feeds
index_stories: index_stories:
- docker exec -it newsblur_web ./manage.py index_stories -R docker exec -it newsblur_web ./manage.py index_stories -R
# performance tests # performance tests
perf-cli: perf-cli:
@ -210,12 +210,12 @@ perf-ui:
locust -f perf/locust.py locust -f perf/locust.py
perf-docker: perf-docker:
- docker build . --file=./perf/Dockerfile --tag=perf-docker docker build . --file=./perf/Dockerfile --tag=perf-docker
- docker run -it -p 8089:8089 perf-docker locust -f locust.py docker run -it -p 8089:8089 perf-docker locust -f locust.py
clean: clean:
- find . -name \*.pyc -delete find . -name \*.pyc -delete
grafana-dashboards: grafana-dashboards:
- python3 utils/grafana_backup.py python3 utils/grafana_backup.py

21
terraform/digitalocean.tf
View file

@ -382,12 +382,12 @@ resource "digitalocean_droplet" "db-redis-sessions" {
} }
resource "digitalocean_droplet" "db-redis-story" { resource "digitalocean_droplet" "db-redis-story" {
count = 2 count = 1
image = var.droplet_os image = var.droplet_os
name = "db-redis-story${count.index + 1}" name = "db-redis-story${count.index + 2}"
region = var.droplet_region region = var.droplet_region
size = contains([1], count.index) ? "m-8vcpu-64gb" : var.redis_story_droplet_size # size = contains([1], count.index) ? "m-8vcpu-64gb" : var.redis_story_droplet_size
# size = var.redis_story_droplet_size size = var.redis_story_droplet_size
ssh_keys = [digitalocean_ssh_key.default.fingerprint] ssh_keys = [digitalocean_ssh_key.default.fingerprint]
provisioner "local-exec" { provisioner "local-exec" {
command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120" command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120"
@ -419,12 +419,13 @@ resource "digitalocean_droplet" "db-redis-pubsub" {
} }
resource "digitalocean_droplet" "db-postgres" { resource "digitalocean_droplet" "db-postgres" {
count = 2 count = 1
backups = true
image = var.droplet_os image = var.droplet_os
name = contains([0], count.index) ? "db-postgres${count.index + 1}" : "db-postgres${count.index + 1}" name = contains([0], count.index) ? "db-postgres${count.index + 2}" : "db-postgres${count.index + 2}"
region = var.droplet_region region = var.droplet_region
size = contains([0], count.index) ? var.droplet_size_160 : var.droplet_size_320 # size = contains([0], count.index) ? var.droplet_size_160 : var.droplet_size_320
# size = var.droplet_size_240 size = var.droplet_size_320
ssh_keys = [digitalocean_ssh_key.default.fingerprint] ssh_keys = [digitalocean_ssh_key.default.fingerprint]
provisioner "local-exec" { provisioner "local-exec" {
command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120" command = "/srv/newsblur/ansible/utils/generate_inventory.py; sleep 120"
@ -491,7 +492,7 @@ resource "digitalocean_droplet" "db-mongo-primary" {
} }
resource "digitalocean_volume" "mongo_secondary_volume" { resource "digitalocean_volume" "mongo_secondary_volume" {
count = 3 count = 2
region = "nyc1" region = "nyc1"
name = "mongosecondary${count.index + 1}" name = "mongosecondary${count.index + 1}"
size = 500 size = 500
@ -500,7 +501,7 @@ resource "digitalocean_volume" "mongo_secondary_volume" {
} }
resource "digitalocean_droplet" "db-mongo-secondary" { resource "digitalocean_droplet" "db-mongo-secondary" {
count = 3 count = 2
# backups = contains([0], count.index) ? true : false # backups = contains([0], count.index) ? true : false
image = var.droplet_os image = var.droplet_os
name = "db-mongo-secondary${count.index + 1}" name = "db-mongo-secondary${count.index + 1}"

2
terraform/variables.tf
View file

@ -89,5 +89,5 @@ variable "elasticsearch_droplet_size" {
variable "redis_story_droplet_size" { variable "redis_story_droplet_size" {
type = string type = string
default = "m-4vcpu-32gb" default = "m-8vcpu-64gb"
} }