From 6476da76a8eeec0a2a48c53eefcaf3eae20c144a Mon Sep 17 00:00:00 2001 From: Dominic Ricottone Date: Tue, 30 Aug 2022 21:58:24 -0500 Subject: [PATCH] Refactoring The site is no more functional now. But the databases are now in separate containers, and there is a working recipe for managing the database initialization/migration (via `make dbinit` and `make dbmigrate`). This should speed up image builds and container starts, and make the entire development cycle faster. Sourcehut files have been moved to `sr/`. --- .gitignore | 7 +- Makefile | 125 ++++++++++++++++-- README.md | 43 +++--- etc/postgresql/postgresql.conf | 2 +- Dockerfile => sr/Dockerfile | 12 +- {etc => sr/etc}/apk/repositories | 0 {etc => sr/etc}/nginx/graphql.conf | 0 {etc => sr/etc}/nginx/headers.conf | 0 {etc => sr/etc}/nginx/http.d/default.conf | 0 {etc => sr/etc}/nginx/http.d/git.conf | 0 {etc => sr/etc}/nginx/http.d/lists.conf | 0 {etc => sr/etc}/nginx/http.d/meta.conf | 0 {etc => sr/etc}/nginx/http.d/test.conf | 0 {etc => sr/etc}/nginx/http.d/todo.conf | 0 {etc => sr/etc}/nginx/nginx.conf | 0 {etc => sr/etc}/nginx/web.conf | 0 {etc => sr/etc}/postfix/generic | 0 {etc => sr/etc}/postfix/main.cf | 0 {etc => sr/etc}/postfix/master.cf | 0 {etc => sr/etc}/postfix/sasl/sasl_passwd | 0 {etc => sr/etc}/postfix/transport | 0 {etc => sr/etc}/sr.ht/config.ini | 28 ++-- .../etc}/supervisor/conf.d/supervisord.conf | 11 -- {usr => sr/usr}/share/nginx/html/index.html | 0 {usr => sr/usr}/share/nginx/html/test.cgi | 0 {usr => sr/usr}/share/nginx/html/test.php | 0 26 files changed, 154 insertions(+), 74 deletions(-) rename Dockerfile => sr/Dockerfile (78%) rename {etc => sr/etc}/apk/repositories (100%) rename {etc => sr/etc}/nginx/graphql.conf (100%) rename {etc => sr/etc}/nginx/headers.conf (100%) rename {etc => sr/etc}/nginx/http.d/default.conf (100%) rename {etc => sr/etc}/nginx/http.d/git.conf (100%) rename {etc => sr/etc}/nginx/http.d/lists.conf (100%) rename {etc => sr/etc}/nginx/http.d/meta.conf (100%) rename {etc => sr/etc}/nginx/http.d/test.conf (100%) rename {etc => sr/etc}/nginx/http.d/todo.conf (100%) rename {etc => sr/etc}/nginx/nginx.conf (100%) rename {etc => sr/etc}/nginx/web.conf (100%) rename {etc => sr/etc}/postfix/generic (100%) rename {etc => sr/etc}/postfix/main.cf (100%) rename {etc => sr/etc}/postfix/master.cf (100%) rename {etc => sr/etc}/postfix/sasl/sasl_passwd (100%) rename {etc => sr/etc}/postfix/transport (100%) rename {etc => sr/etc}/sr.ht/config.ini (81%) rename {etc => sr/etc}/supervisor/conf.d/supervisord.conf (74%) rename {usr => sr/usr}/share/nginx/html/index.html (100%) rename {usr => sr/usr}/share/nginx/html/test.cgi (100%) rename {usr => sr/usr}/share/nginx/html/test.php (100%) diff --git a/.gitignore b/.gitignore index 7e73b0e..7f62973 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -etc/sr.ht/pgp.key -etc/sr.ht/pgp.pubkey -postgresql-data -redis-data +sr/etc/sr.ht/pgp.key +sr/etc/sr.ht/pgp.pubkey +postgres diff --git a/Makefile b/Makefile index 3adb176..cc3a1d0 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,117 @@ +# set docker-compliant container management CLI binary +CONMAN=docker + +# set fun names for the containers +HUTTESE_NETWORK=holonet +HUTTESE_SRHT=huttsr +HUTTESE_REDIS=huttredis +HUTTESE_POSTGRES=huttpg + +# configure redis +REDIS_TARGET=redis:alpine3.15 +REDIS_LOCALNAME=my-redis +REDIS_DATADIR=path/to/redis/db + +# configure postgres +POSTGRES_TARGET=postgres:alpine3.15 +POSTGRES_LOCALNAME=my-postgres +POSTGRES_DATADIR=/home/al_dente/dev/huttese/postgres +POSTGRES_CONF=/home/al_dente/dev/huttese/etc/postgresql/postgresql.conf + +# set image tag data +SRHT_LOCALNAME=srht +SRHT_LOCALVERSION=1 + +cleanup: + $(CONMAN) network disconnect --force $(HUTTESE_NETWORK) $(HUTTESE_REDIS) >/dev/null 2>&1 || true + $(CONMAN) network disconnect --force $(HUTTESE_NETWORK) $(HUTTESE_POSTGRES) >/dev/null 2>&1 || true + + $(CONMAN) rm --force $(HUTTESE_REDIS) >/dev/null 2>&1 || true + $(CONMAN) image rm --force $(REDIS_LOCALNAME):latest >/dev/null 2>&1 || true + + $(CONMAN) rm --force $(HUTTESE_POSTGRES) >/dev/null 2>&1 || true + $(CONMAN) image rm --force $(POSTGRES_LOCALNAME):latest >/dev/null 2>&1 || true + +setup: + $(CONMAN) inspect $(HUTTESE_NETWORK) >/dev/null 2>&1 \ + || $(CONMAN) network create $(HUTTESE_NETWORK) + + $(CONMAN) inspect $(REDIS_LOCALNAME) >/dev/null 2>&1 \ + || $(CONMAN) pull $(REDIS_TARGET) \ + && $(CONMAN) tag $(REDIS_TARGET) $(REDIS_LOCALNAME) + $(CONMAN) run --detach --name $(HUTTESE_REDIS) --restart always \ + $(REDIS_LOCALNAME) + #if I need persistence later: + # --mount type=bind,src=$(REDIS_DATADIR),dst=/data \ + # $(REDIS_LOCALNAME) redis-server --save 60 1 --loglevel warning + $(CONMAN) network connect --alias $(HUTTESE_REDIS) \ + $(HUTTESE_NETWORK) $(HUTTESE_REDIS) + # redis is now available at redis://huttredis:6379 + + $(CONMAN) inspect $(POSTGRES_LOCALNAME) >/dev/null 2>&1 \ + || $(CONMAN) pull $(POSTGRES_TARGET) \ + && $(CONMAN) tag $(POSTGRES_TARGET) $(POSTGRES_LOCALNAME) + $(CONMAN) run --detach --name $(HUTTESE_POSTGRES) --restart always \ + --env POSTGRES_HOST_AUTH_METHOD=trust \ + --mount type=bind,src=$(POSTGRES_DATADIR),dst=/var/lib/postgresql/data \ + --mount type=bind,src=$(POSTGRES_CONF),dst=/etc/postgresql/postgresql.conf \ + $(POSTGRES_LOCALNAME) -c 'config_file=/etc/postgresql/postgresql.conf' + $(CONMAN) network connect --alias $(HUTTESE_POSTGRES) \ + $(HUTTESE_NETWORK) $(HUTTESE_POSTGRES) + # postgres is now available at postgresql://postgres@huttpg:5432 + image: - docker build . --tag tatooine + $(CONMAN) inspect $(SRHT_LOCALNAME) >/dev/null 2>&1 \ + || $(CONMAN) build \ + --tag $(SRHT_LOCALNAME):latest \ + --tag $(SRHT_LOCALNAME):$(SRHT_LOCALVERSION) \ + sr/ -clean: - docker rm --force tatooine-dev +dbinit: image + $(CONMAN) inspect $(HUTTESE_NETWORK) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 -run: - docker run -it --name tatooine-dev \ - --hostname tatooine -p 80:8080 \ - --mount type=bind,src=/home/al_dente/dev/huttese/redis-data,dst=/data \ - --mount type=bind,src=/home/al_dente/dev/huttese/postgresql-data,dst=/var/lib/postgresql/data \ - tatooine + $(CONMAN) exec \ + $(HUTTESE_POSTGRES) createdb -U postgres meta.sr.ht + $(CONMAN) run --name $(HUTTESE_SRHT)_dbinit \ + --network $(HUTTESE_NETWORK) \ + $(SRHT_LOCALNAME) metasrht-initdb + $(CONMAN) rm $(HUTTESE_SRHT)_dbinit + +dbmigrate: image + $(CONMAN) inspect $(HUTTESE_NETWORK) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 + + $(CONMAN) run --name $(HUTTESE_SRHT)_dbmigrate \ + --network $(HUTTESE_NETWORK) \ + $(SRHT_LOCALNAME) srht-migrate meta.sr.ht -a upgrade head + $(CONMAN) run --name $(HUTTESE_SRHT)_dbmigrate \ + --network $(HUTTESE_NETWORK) \ + $(SRHT_LOCALNAME) metasrht-migrate -a upgrade head + $(CONMAN) rm $(HUTTESE_SRHT)_dbmigrate + +start: image + $(CONMAN) inspect $(HUTTESE_NETWORK) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_REDIS) >/dev/null 2>&1 -start: - docker run --detach --name tatooine-dev \ + $(CONMAN) run --detach --name $(HUTTESE_SRHT) --restart always \ --hostname tatooine -p 80:8080 \ - --mount type=bind,src=/home/al_dente/dev/huttese/redis-data,dst=/data \ - --mount type=bind,src=/home/al_dente/dev/huttese/postgresql-data,dst=/var/lib/postgresql/data \ - tatooine + $(SRHT_LOCALNAME) + $(CONMAN) network connect --alias $(HUTTESE_SRHT) \ + $(HUTTESE_NETWORK) $(HUTTESE_SRHT) + +stop: + $(CONMAN) stop $(SRHT_LOCALNAME) + +restart: + $(CONMAN) inspect $(HUTTESE_NETWORK) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 + $(CONMAN) inspect -f '{{.State.Running}}' $(HUTTESE_REDIS) >/dev/null 2>&1 + + $(CONMAN) restart $(SRHT_LOCALNAME) + +clean: + $(CONMAN) rm --force $(SRHT_LOCALNAME) >/dev/null 2>&1 || true + $(CONMAN) image rm --force $(SRHT_LOCALNAME):latest >/dev/null 2>&1 || true diff --git a/README.md b/README.md index 91de504..5c4057c 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,44 @@ -## Usage +# huttese -Disable the haproxy docker container. +The image is built in `sr/`. -Paste the following into the hosts file: +Volume mounts are in the top-level directory. -``` -127.0.0.1 git.intra.dominic-ricottone.com lists.intra.dominic-ricottone.com meta.intra.dominic-ricottone.com todo.intra.dominic-ricottone.com -``` - -Run `make clean && make run`. +Makefile is in the top-level directory. ## To-Do -nginx proxying is not working +nginx proxying is not working. + +need metasrht service to become accessible so that i can register oauth for git, lists, todo services. -need metasrht service to become accessible so that i can register oauth for git, lists, todo services +should also setup the metasrht-webhook process (`celery -A metasrht.webhooks worker --loglevel=info`). -should also setup the metasrht-webhook process (celery -A metasrht.webhooks worker --loglevel=info) +maybe should run migration (`metasrht-migrate -a upgrade head`) every time? -maybe should run migration (metasrht-migrate -a upgrade head) every time? +probably should not be running most of these services as `root`. +[apkbuilds repo](https://git.sr.ht/~sircmpwn/sr.ht-apkbuilds/tree) indicates that metasrht (and friends) run as `meta`. +Probably have `git`, `lists`, `todo`, and `build` users. -most importantly, need to figure out how to secure pgp keys, worker private key, network private key, and webhook private key +most importantly, need to figure out how to secure pgp keys, worker private key, network private key, and webhook private key. + +cleanup the postgres files, probably to a `pg` directory. ## Installation -Run `make image && make start`. +Disable the haproxy docker container. -On first run, these need to be done manually. Or at least after the postgres database is running. +Paste the following into the hosts file: ``` -su - postgres -initdb /var/lib/postgresql/data -createdb -U postgres meta.sr.ht -metasrht-initdb +127.0.0.1 git.intra.dominic-ricottone.com lists.intra.dominic-ricottone.com meta.intra.dominic-ricottone.com todo.intra.dominic-ricottone.com ``` +Run `make setup && make image && make dbinit && make start`. + +For subsequent use, run `make image && make dbmigrate && make start`. + +Or to just restart the service if stopped, try `make restart`. + diff --git a/etc/postgresql/postgresql.conf b/etc/postgresql/postgresql.conf index 963ca63..74b49b9 100644 --- a/etc/postgresql/postgresql.conf +++ b/etc/postgresql/postgresql.conf @@ -53,7 +53,7 @@ ident_file = '/var/lib/postgresql/data/pg_ident.conf' # ident configuration fi # - Connection Settings - -listen_addresses = 'localhost' # comma-separated list of addresses; defaults to 'localhost'; use '*' for all +listen_addresses = 'huttpg' # comma-separated list of addresses; defaults to 'localhost'; use '*' for all port = 5432 #max_connections = 100 #superuser_reserved_connections = 3 diff --git a/Dockerfile b/sr/Dockerfile similarity index 78% rename from Dockerfile rename to sr/Dockerfile index 38aab1f..289b4a1 100644 --- a/Dockerfile +++ b/sr/Dockerfile @@ -2,13 +2,10 @@ FROM alpine:3.15 COPY etc/apk/repositories /etc/apk/repositories RUN wget --quiet --output-document=/etc/apk/keys/alpine@sr.ht.rsa.pub https://mirror.sr.ht/alpine/alpine@sr.ht.rsa.pub RUN apk update -RUN apk add redis postgresql14 postfix meta.sr.ht git.sr.ht todo.sr.ht supervisor nginx fcgiwrap spawn-fcgi py3-gunicorn celery +RUN apk add postfix meta.sr.ht git.sr.ht todo.sr.ht supervisor nginx fcgiwrap spawn-fcgi py3-gunicorn # setup directories RUN mkdir /etc/postfix/sasl && chmod 700 /etc/postfix/sasl -RUN mkdir /data -RUN mkdir /var/lib/postgresql/data && chown postgres:postgres /var/lib/postgresql/data && chmod 750 /var/lib/postgresql/data -RUN mkdir /run/postgresql && chown postgres:postgres /run/postgresql RUN mkdir /var/log/supervisord # setup nginx @@ -36,13 +33,6 @@ RUN postmap /etc/postfix/generic COPY etc/postfix/sasl/sasl_passwd /etc/postfix/sasl/sasl_passwd RUN postmap /etc/postfix/sasl/sasl_passwd -# setup redis -VOLUME /data - -# setup postgresql -VOLUME /var/lib/postgresql/data -COPY --chown=postgres:postgres etc/postgresql/postgresql.conf /etc/postgresql/postgresql.conf - # setup sourcehut COPY etc/sr.ht/config.ini /etc/sr.ht/config.ini COPY etc/sr.ht/pgp.key /etc/sr.ht/pgp.key diff --git a/etc/apk/repositories b/sr/etc/apk/repositories similarity index 100% rename from etc/apk/repositories rename to sr/etc/apk/repositories diff --git a/etc/nginx/graphql.conf b/sr/etc/nginx/graphql.conf similarity index 100% rename from etc/nginx/graphql.conf rename to sr/etc/nginx/graphql.conf diff --git a/etc/nginx/headers.conf b/sr/etc/nginx/headers.conf similarity index 100% rename from etc/nginx/headers.conf rename to sr/etc/nginx/headers.conf diff --git a/etc/nginx/http.d/default.conf b/sr/etc/nginx/http.d/default.conf similarity index 100% rename from etc/nginx/http.d/default.conf rename to sr/etc/nginx/http.d/default.conf diff --git a/etc/nginx/http.d/git.conf b/sr/etc/nginx/http.d/git.conf similarity index 100% rename from etc/nginx/http.d/git.conf rename to sr/etc/nginx/http.d/git.conf diff --git a/etc/nginx/http.d/lists.conf b/sr/etc/nginx/http.d/lists.conf similarity index 100% rename from etc/nginx/http.d/lists.conf rename to sr/etc/nginx/http.d/lists.conf diff --git a/etc/nginx/http.d/meta.conf b/sr/etc/nginx/http.d/meta.conf similarity index 100% rename from etc/nginx/http.d/meta.conf rename to sr/etc/nginx/http.d/meta.conf diff --git a/etc/nginx/http.d/test.conf b/sr/etc/nginx/http.d/test.conf similarity index 100% rename from etc/nginx/http.d/test.conf rename to sr/etc/nginx/http.d/test.conf diff --git a/etc/nginx/http.d/todo.conf b/sr/etc/nginx/http.d/todo.conf similarity index 100% rename from etc/nginx/http.d/todo.conf rename to sr/etc/nginx/http.d/todo.conf diff --git a/etc/nginx/nginx.conf b/sr/etc/nginx/nginx.conf similarity index 100% rename from etc/nginx/nginx.conf rename to sr/etc/nginx/nginx.conf diff --git a/etc/nginx/web.conf b/sr/etc/nginx/web.conf similarity index 100% rename from etc/nginx/web.conf rename to sr/etc/nginx/web.conf diff --git a/etc/postfix/generic b/sr/etc/postfix/generic similarity index 100% rename from etc/postfix/generic rename to sr/etc/postfix/generic diff --git a/etc/postfix/main.cf b/sr/etc/postfix/main.cf similarity index 100% rename from etc/postfix/main.cf rename to sr/etc/postfix/main.cf diff --git a/etc/postfix/master.cf b/sr/etc/postfix/master.cf similarity index 100% rename from etc/postfix/master.cf rename to sr/etc/postfix/master.cf diff --git a/etc/postfix/sasl/sasl_passwd b/sr/etc/postfix/sasl/sasl_passwd similarity index 100% rename from etc/postfix/sasl/sasl_passwd rename to sr/etc/postfix/sasl/sasl_passwd diff --git a/etc/postfix/transport b/sr/etc/postfix/transport similarity index 100% rename from etc/postfix/transport rename to sr/etc/postfix/transport diff --git a/etc/sr.ht/config.ini b/sr/etc/sr.ht/config.ini similarity index 81% rename from etc/sr.ht/config.ini rename to sr/etc/sr.ht/config.ini index b9ffc9b..bc33e6a 100644 --- a/etc/sr.ht/config.ini +++ b/sr/etc/sr.ht/config.ini @@ -20,7 +20,7 @@ service-key=REDACTED # try: `srht-keygen network` network-key=REDACTED -redis-host=redis://localhost +redis-host=redis://huttredis [objects] @@ -64,12 +64,12 @@ post-update-script=/usr/bin/gitsrht-update-hook outgoing-domain=tatooine # SQLAlchemy connection string -#connection-string=postgresql://postgres@localhost/git.sr.ht -connection-string=postgresql://postgres@localhost/git.sr.ht?sslmode=disable +#connection-string=postgresql://postgres@huttpg:5432/git.sr.ht +connection-string=postgresql://postgres@huttpg:5432/git.sr.ht?sslmode=disable migrate-on-upgrade=yes # Webhooks connection string -webhooks=redis://localhost:6379/1 +webhooks=redis://huttredis:6379/1 s3-bucket= s3-prefix= @@ -101,15 +101,15 @@ debug-host=0.0.0.0 debug-port=5006 # SQLAlchemy connection string -#connection-string=postgresql://postgres@localhost/lists.sr.ht -connection-string=postgresql://postgres@localhost/lists.sr.ht?sslmode=disable +#connection-string=postgresql://postgres@huttpg:5432/lists.sr.ht +connection-string=postgresql://postgres@huttpg:5432/lists.sr.ht?sslmode=disable migrate-on-upgrade=yes # Webhooks connection string -webhooks=redis://localhost:6379/1 +webhooks=redis://huttredis:6379/1 # Celery connection string -redis=redis://localhost:6379/0 +redis=redis://huttredis:6379/0 # Trusted upstream SMTP server generating Authentication-Results header fields msgauth-server=tatooine @@ -146,12 +146,12 @@ debug-host=0.0.0.0 debug-port=5000 # SQLAlchemy connection string -#connection-string=postgresql://postgres@localhost/meta.sr.ht -connection-string=postgresql://postgres@localhost/meta.sr.ht?sslmode=disable +#connection-string=postgresql://postgres@huttpg:5432/meta.sr.ht +connection-string=postgresql://postgres@huttpg:5432/meta.sr.ht?sslmode=disable migrate-on-upgrade=yes # Webhooks connection string -webhooks=redis://localhost:6379/1 +webhooks=redis://huttredis:6379/1 [meta.sr.ht::api] @@ -200,11 +200,11 @@ oauth-client-secret= notify-from=hutt@tatooine # SQLAlchemy connection string -#connection-string=postgresql://postgres@localhost/todo.sr.ht -connection-string=postgresql://postgres@localhost/todo.sr.ht?sslmode=disable +#connection-string=postgresql://postgres@huttpg:5432/todo.sr.ht +connection-string=postgresql://postgres@huttpg:5432/todo.sr.ht?sslmode=disable migrate-on-upgrade=yes # Webhooks connection string -webhooks=redis://localhost:6379/1 +webhooks=redis://huttredis:6379/1 diff --git a/etc/supervisor/conf.d/supervisord.conf b/sr/etc/supervisor/conf.d/supervisord.conf similarity index 74% rename from etc/supervisor/conf.d/supervisord.conf rename to sr/etc/supervisor/conf.d/supervisord.conf index 88103f1..a139a9c 100644 --- a/etc/supervisor/conf.d/supervisord.conf +++ b/sr/etc/supervisor/conf.d/supervisord.conf @@ -16,17 +16,6 @@ command=postfix start startsecs=0 redirect_stderr=true -[program:postgresql] -autorestart=true -command=postgres -c config_file=/etc/postgresql/postgresql.conf -redirect_stderr=true -user=postgres - -[program:redis] -autorestart=true -command=redis-server --save 60 1 --loglevel warning -redirect_stderr=true - [program:spawn-fcgi] autorestart=true command=spawn-fcgi -n -u nginx -p 9000 -- /usr/bin/fcgiwrap -f diff --git a/usr/share/nginx/html/index.html b/sr/usr/share/nginx/html/index.html similarity index 100% rename from usr/share/nginx/html/index.html rename to sr/usr/share/nginx/html/index.html diff --git a/usr/share/nginx/html/test.cgi b/sr/usr/share/nginx/html/test.cgi similarity index 100% rename from usr/share/nginx/html/test.cgi rename to sr/usr/share/nginx/html/test.cgi diff --git a/usr/share/nginx/html/test.php b/sr/usr/share/nginx/html/test.php similarity index 100% rename from usr/share/nginx/html/test.php rename to sr/usr/share/nginx/html/test.php -- 2.45.2