From 0ae257ccf8378d123c37e055d473273b4a2091bb Mon Sep 17 00:00:00 2001 From: Dominic Ricottone Date: Wed, 7 Sep 2022 00:02:08 -0500 Subject: [PATCH] Progress on podman process The podman process is now very mature, and some things can actually be copied over to the original docker process. Sourcehut now is fully functional, with possibly exception for the mail service. Needs to be tested. Furthermore, the cron jobs should be configured to run. x86/aarch64 simultaneous support is also complete. See the sister repo, huttese-apk. --- .gitignore | 2 + Makefile.podman | 33 ++++++++----- README.md | 44 +++++++++-------- sr/Dockerfile.aarch64 | 49 +++++++++++++++++++ sr/{Dockerfile => Dockerfile.x86} | 2 +- sr/etc/apk/repositories.aarch64 | 3 ++ sr/etc/apk/{repositories => repositories.x86} | 0 sr/etc/nginx/http.d/git.conf | 10 +++- sr/etc/nginx/http.d/lists.conf | 8 ++- sr/etc/nginx/http.d/meta.conf | 10 +++- sr/etc/nginx/http.d/test.conf | 4 +- sr/etc/nginx/http.d/todo.conf | 8 ++- sr/etc/sr.ht/config.ini | 7 +-- 13 files changed, 133 insertions(+), 47 deletions(-) create mode 100644 sr/Dockerfile.aarch64 rename sr/{Dockerfile => Dockerfile.x86} (97%) create mode 100644 sr/etc/apk/repositories.aarch64 rename sr/etc/apk/{repositories => repositories.x86} (100%) diff --git a/.gitignore b/.gitignore index ffd4434..a83bafe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ sr/etc/sr.ht/pgp.key sr/etc/sr.ht/pgp.pubkey **/datadir +Makefile +sr/Dockerfile diff --git a/Makefile.podman b/Makefile.podman index e72e28c..7cfb5ac 100644 --- a/Makefile.podman +++ b/Makefile.podman @@ -8,12 +8,12 @@ HUTTESE_REDIS=huttredis HUTTESE_POSTGRES=huttpg # configure redis -REDIS_TARGET=redis:alpine3.15 +REDIS_TARGET=docker.io/library/redis:alpine3.15 REDIS_LOCALNAME=my-redis REDIS_DATADIR=/var/deploy/data/redis # configure postgres -POSTGRES_TARGET=postgres:alpine3.15 +POSTGRES_TARGET=docker.io/library/postgres:alpine3.15 POSTGRES_LOCALNAME=my-postgres POSTGRES_DATADIR=/var/deploy/data/postgres POSTGRES_CONF=/var/deploy/conf/postgres/postgresql.conf @@ -30,6 +30,11 @@ CERTDIR=/var/deploy/certs pod: $(PODMAN) pod exists $(HUTTESE_POD) >/dev/null 2>&1 \ || $(PODMAN) pod create --name $(HUTTESE_POD) \ + --add-host=tatooine:127.0.0.1 \ + --add-host=dominic-ricottone.com:127.0.0.1 \ + --add-host=meta.dominic-ricottone.com:127.0.0.1 \ + --add-host=git.dominic-ricottone.com:127.0.0.1 \ + --add-host=todo.dominic-ricottone.com:127.0.0.1 \ --publish 0.0.0.0:80:80 --publish 0.0.0.0:443:443 cleanup: @@ -49,7 +54,7 @@ setup: pod #if I need persistence later: # --mount type=bind,src=$(REDIS_DATADIR),dst=/data \ # $(REDIS_LOCALNAME) redis-server --save 60 1 --loglevel warning - # redis is now available at redis://huttredis:6379 + # redis is now available at redis://localhost:6379 $(PODMAN) inspect $(POSTGRES_LOCALNAME) >/dev/null 2>&1 \ || $(PODMAN) pull $(POSTGRES_TARGET) \ @@ -57,18 +62,24 @@ setup: pod $(PODMAN) run --detach --name $(HUTTESE_POSTGRES) --restart always \ --env POSTGRES_HOST_AUTH_METHOD=trust \ --pod podracing \ - --mount type=bind,src=$(POSTGRES_DATADIR),dst=/var/lib/postgresql/data \ - --mount type=bind,src=$(POSTGRES_CONF),dst=/etc/postgresql/postgresql.conf \ + --mount type=bind,src=$(POSTGRES_DATADIR),dst=/var/lib/postgresql/data,Z \ + --mount type=bind,src=$(POSTGRES_CONF),dst=/etc/postgresql/postgresql.conf,Z \ $(POSTGRES_LOCALNAME) -c 'config_file=/etc/postgresql/postgresql.conf' - # postgres is now available at postgresql://postgres@huttpg:5432 + # postgres is now available at postgresql://postgres@localhost:5432 image: + # these are only necessary if on aarch64 + cp -r /var/deploy/build/huttese-apk/pkg sr/pkg + cp -r /var/deploy/build/huttese-apk/keys sr/etc/apk/keys + $(PODMAN) inspect $(SRHT_LOCALNAME) >/dev/null 2>&1 \ || $(PODMAN) build \ --tag $(SRHT_LOCALNAME):latest \ --tag $(SRHT_LOCALNAME):$(SRHT_LOCALVERSION) \ sr/ + rm -rf sr/pkg sr/etc/apk/keys + dbinit: pod image $(PODMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 $(PODMAN) exec $(HUTTESE_POSTGRES) \ @@ -84,7 +95,6 @@ dbinit: pod image $(PODMAN) run --name $(HUTTESE_SRHT)_dbinit --rm \ --pod podracing \ - $(SRHT_LOCALNAME) metasrht-initdb $(SRHT_LOCALNAME) gitsrht-initdb $(PODMAN) run --name $(HUTTESE_SRHT)_dbinit --rm \ @@ -124,10 +134,9 @@ start: pod image $(PODMAN) inspect -f '{{.State.Running}}' $(HUTTESE_REDIS) >/dev/null 2>&1 $(PODMAN) run --detach --name $(HUTTESE_SRHT) --restart always \ - --hostname tatooine --hostname dominic-ricottone.com \ --pod podracing \ - --mount type=bind,src=$(GIT_DATADIR),dst=/var/lib/git \ - --mount type=bind,src=$(CERTDIR),dst=/var/lets-encrypt \ + --mount type=bind,src=$(GIT_DATADIR),dst=/var/lib/git,Z \ + --mount type=bind,src=$(CERTDIR),dst=/var/lets-encrypt,Z \ $(SRHT_LOCALNAME) USER_EMAIL?= @@ -146,13 +155,13 @@ shell: $(PODMAN) exec -it $(HUTTESE_SRHT) sh stop: - $(PODMAN) stop $(SRHT_LOCALNAME) + $(PODMAN) stop $(HUTTESE_SRHT) restart: $(PODMAN) inspect -f '{{.State.Running}}' $(HUTTESE_POSTGRES) >/dev/null 2>&1 $(PODMAN) inspect -f '{{.State.Running}}' $(HUTTESE_REDIS) >/dev/null 2>&1 - $(PODMAN) restart $(SRHT_LOCALNAME) + $(PODMAN) restart $(HUTTESE_SRHT) clean: $(PODMAN) rm --force $(HUTTESE_SRHT) >/dev/null 2>&1 || true diff --git a/README.md b/README.md index f5ab01a..bca0a04 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,44 @@ # huttese -The image is built in `sr/`. +Sourcehut is only officially supported on x86 Alpine Linux. -Volume mounts are in the top-level directory. +I say **challenge accepted**. +This repo walks through the process of running Sourcehut within containers (either `docker` or `podman`, possibly other runtimes as well) and even on ARM processors. -Makefile is in the top-level directory. +## Usage -## To-Do - -nginx proxying is working sometimes. +Select the recipes for your container runtime: `Makefile.docker` or `Makefile.podman`. +`ln -s Makefile.selection Makefile` to get going from there. -need metasrht service to become accessible so that i can register oauth for git, lists, todo services. +Select the image for your architecture: `Dockerfile.x86` or `dockerfile.aarch64`. +`ln -s Dockerfile.selection Dockerfile` to proceed. -should also setup the metasrht-webhook process (`celery -A metasrht.webhooks worker --loglevel=info`). +Note that hosting on an ARM machine will additionally require building the APKs. +See huttese-apk for more information. -maybe should run migration (`metasrht-migrate -a upgrade head`) every time? +Run `make setup && make image && make dbinit && make start`. -most importantly, need to figure out how to secure pgp keys, worker private key, network private key, and webhook private key. +For subsequent use, run `make image && make dbmigrate && make start`. -cleanup the postgres files, probably to a `pg` directory. +Or to just restart the service if stopped (as by `make stop`), try `make restart`. -for now, not targeting listssrht. may add later. +To clean up the main container, use `make clean`. +To clean up all containers, use `make clean && make cleanup`. +If you even want to throw out the postgres database, try `sudo rm -f pg/datadir`. -## Installation -Disable the haproxy docker container. +## To-Do -Paste the following into the hosts file: +Copy some notable improvements from podman process to docker process. -``` -127.0.0.1 git.intra.dominic-ricottone.com meta.intra.dominic-ricottone.com todo.intra.dominic-ricottone.com test.intra.dominic-ricottone.com -``` +For podman process, add cron jobs as systemd timers on the host system. -Run `make setup && make image && make dbinit && make start`. +Need to figure out how to secure pgp keys, worker private key, network private key, and webhook private key. +Currently just manually cutting from commits. -For subsequent use, run `make image && make dbmigrate && make start`. +Consider a `configure` script to manage the selection of `Makefile.*`, `Dockerfile.*`. -Or to just restart the service if stopped, try `make restart`. +Consider `lists.sr.ht`, `builds.sr.ht`. diff --git a/sr/Dockerfile.aarch64 b/sr/Dockerfile.aarch64 new file mode 100644 index 0000000..62a030b --- /dev/null +++ b/sr/Dockerfile.aarch64 @@ -0,0 +1,49 @@ +FROM alpine:3.15 +COPY etc/apk/repositories.aarch64 /etc/apk/repositories +COPY pkg /var/huttese-apk +COPY etc/apk/keys/me@dominic-ricottone.com.rsa.pub /etc/apk/keys/me@dominic-ricottone.com.rsa.pub +RUN apk update +RUN apk add postfix meta.sr.ht git.sr.ht todo.sr.ht supervisor nginx fcgiwrap spawn-fcgi py3-gunicorn + +# setup directories +RUN mkdir /etc/postfix/sasl && chmod 700 /etc/postfix/sasl +RUN mkdir /var/log/supervisord + +# setup nginx +COPY etc/nginx/nginx.conf /etc/nginx/nginx.conf +COPY etc/nginx/graphql.conf /etc/nginx/graphql.conf +COPY etc/nginx/headers.conf /etc/nginx/headers.conf +COPY etc/nginx/web.conf /etc/nginx/web.conf +COPY etc/nginx/http.d/test.conf /etc/nginx/http.d/test.conf +COPY --chown=nginx:nginx usr/share/nginx/html/index.html /usr/share/nginx/html/index.html +COPY --chown=nginx:nginx usr/share/nginx/html/test.cgi /usr/share/nginx/html/test.cgi +COPY --chown=nginx:nginx usr/share/nginx/html/test.php /usr/share/nginx/html/test.php +COPY etc/nginx/http.d/default.conf /etc/nginx/http.d/default.conf +COPY etc/nginx/http.d/git.conf /etc/nginx/http.d/git.conf +#COPY etc/nginx/http.d/lists.conf /etc/nginx/http.d/lists.conf +COPY etc/nginx/http.d/meta.conf /etc/nginx/http.d/meta.conf +COPY etc/nginx/http.d/todo.conf /etc/nginx/http.d/todo.conf + +# setup postfix +COPY etc/postfix/main.cf /etc/postfix/main.cf +COPY etc/postfix/master.cf /etc/postfix/master.cf +COPY etc/postfix/transport /etc/postfix/transport +RUN postmap /etc/postfix/transport +COPY etc/postfix/generic /etc/postfix/generic +RUN postmap /etc/postfix/generic +COPY etc/postfix/sasl/sasl_passwd /etc/postfix/sasl/sasl_passwd +RUN postmap /etc/postfix/sasl/sasl_passwd + +# setup sourcehut +VOLUME /var/lib/git +COPY etc/sr.ht/config.ini /etc/sr.ht/config.ini +COPY etc/sr.ht/pgp.key /etc/sr.ht/pgp.key +COPY etc/sr.ht/pgp.pubkey /etc/sr.ht/pgp.pubkey + +# setup supervisor +USER root +EXPOSE 80 +EXPOSE 443 +COPY etc/supervisor/conf.d/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +CMD ["/usr/bin/supervisord","-c","/etc/supervisor/conf.d/supervisord.conf"] + diff --git a/sr/Dockerfile b/sr/Dockerfile.x86 similarity index 97% rename from sr/Dockerfile rename to sr/Dockerfile.x86 index 5ca4335..78d7c87 100644 --- a/sr/Dockerfile +++ b/sr/Dockerfile.x86 @@ -1,5 +1,5 @@ FROM alpine:3.15 -COPY etc/apk/repositories /etc/apk/repositories +COPY etc/apk/repositories.x86 /etc/apk/repositories RUN wget --quiet --output-document=/etc/apk/keys/alpine@sr.ht.rsa.pub https://mirror.sr.ht/alpine/alpine@sr.ht.rsa.pub RUN apk update RUN apk add postfix meta.sr.ht git.sr.ht todo.sr.ht supervisor nginx fcgiwrap spawn-fcgi py3-gunicorn diff --git a/sr/etc/apk/repositories.aarch64 b/sr/etc/apk/repositories.aarch64 new file mode 100644 index 0000000..ccdf3e6 --- /dev/null +++ b/sr/etc/apk/repositories.aarch64 @@ -0,0 +1,3 @@ +/var/huttese-apk +https://dl-cdn.alpinelinux.org/alpine/v3.15/main +https://dl-cdn.alpinelinux.org/alpine/v3.15/community diff --git a/sr/etc/apk/repositories b/sr/etc/apk/repositories.x86 similarity index 100% rename from sr/etc/apk/repositories rename to sr/etc/apk/repositories.x86 diff --git a/sr/etc/nginx/http.d/git.conf b/sr/etc/nginx/http.d/git.conf index 4d9e0b1..fd7cf61 100644 --- a/sr/etc/nginx/http.d/git.conf +++ b/sr/etc/nginx/http.d/git.conf @@ -21,8 +21,8 @@ server { gzip_types text/css text/html; server_name git.dominic-ricottone.com; - ssl_certificate /var/lets-encrypt/inter.dominic-ricottone.com.cert; - ssl_certificate_key /var/lets-encrypt/inter.dominic-ricottone.com.key; + ssl_certificate /var/lets-encrypt/inter.lets-encrypt.cert; + ssl_certificate_key /var/lets-encrypt/inter.lets-encrypt.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; ssl_prefer_server_ciphers on; @@ -30,6 +30,8 @@ server { client_max_body_size 100M; location / { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5001; include headers.conf; add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src * data:; script-src 'self' 'unsafe-inline'" always; @@ -37,6 +39,8 @@ server { } location /query { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5101; include graphql.conf; } @@ -47,6 +51,8 @@ server { } location = /authorize { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5001; proxy_pass_request_body off; proxy_set_header Content-Length ""; diff --git a/sr/etc/nginx/http.d/lists.conf b/sr/etc/nginx/http.d/lists.conf index ec5d2ba..d67f928 100644 --- a/sr/etc/nginx/http.d/lists.conf +++ b/sr/etc/nginx/http.d/lists.conf @@ -21,8 +21,8 @@ server { gzip_types text/css text/html; server_name lists.dominic-ricottone.com; - ssl_certificate /var/lets-encrypt/inter.dominic-ricottone.com.cert; - ssl_certificate_key /var/lets-encrypt/inter.dominic-ricottone.com.key; + ssl_certificate /var/lets-encrypt/inter.lets-encrypt.cert; + ssl_certificate_key /var/lets-encrypt/inter.lets-encrypt.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; ssl_prefer_server_ciphers on; @@ -30,6 +30,8 @@ server { client_max_body_size 100M; location / { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5006; include headers.conf; add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; script-src 'self' 'unsafe-inline'"; @@ -37,6 +39,8 @@ server { } location /query { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5106; include graphql.conf; } diff --git a/sr/etc/nginx/http.d/meta.conf b/sr/etc/nginx/http.d/meta.conf index d5c86e3..05ea137 100644 --- a/sr/etc/nginx/http.d/meta.conf +++ b/sr/etc/nginx/http.d/meta.conf @@ -21,13 +21,15 @@ server { gzip_types text/css text/html; server_name meta.dominic-ricottone.com; - ssl_certificate /var/lets-encrypt/inter.dominic-ricottone.com.cert; - ssl_certificate_key /var/lets-encrypt/inter.dominic-ricottone.com.key; + ssl_certificate /var/lets-encrypt/inter.lets-encrypt.cert; + ssl_certificate_key /var/lets-encrypt/inter.lets-encrypt.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; ssl_prefer_server_ciphers on; location / { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5000; include headers.conf; add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; script-src 'self' 'unsafe-inline' *.stripe.com *.stripe.network; frame-src *.stripe.com *.stripe.network" always; @@ -35,11 +37,15 @@ server { } location /register { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5000; add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; script-src 'self' 'unsafe-inline' *.stripe.com *.stripe.network; frame-src *.stripe.com *.stripe.network" always; } location /query { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5100; include graphql.conf; } diff --git a/sr/etc/nginx/http.d/test.conf b/sr/etc/nginx/http.d/test.conf index ce7cd10..64e0b22 100644 --- a/sr/etc/nginx/http.d/test.conf +++ b/sr/etc/nginx/http.d/test.conf @@ -1,8 +1,8 @@ server { listen 80 default_server; listen 443 default_server; - ssl_certificate /var/lets-encrypt/inter.dominic-ricottone.com.cert; - ssl_certificate_key /var/lets-encrypt/inter.dominic-ricottone.com.key; + ssl_certificate /var/lets-encrypt/inter.lets-encrypt.cert; + ssl_certificate_key /var/lets-encrypt/inter.lets-encrypt.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; ssl_prefer_server_ciphers on; diff --git a/sr/etc/nginx/http.d/todo.conf b/sr/etc/nginx/http.d/todo.conf index c65b29f..3ccff7a 100644 --- a/sr/etc/nginx/http.d/todo.conf +++ b/sr/etc/nginx/http.d/todo.conf @@ -21,8 +21,8 @@ server { gzip_types text/css text/html; server_name todo.dominic-ricottone.com; - ssl_certificate /var/lets-encrypt/inter.dominic-ricottone.com.cert; - ssl_certificate_key /var/lets-encrypt/inter.dominic-ricottone.com.key; + ssl_certificate /var/lets-encrypt/inter.lets-encrypt.cert; + ssl_certificate_key /var/lets-encrypt/inter.lets-encrypt.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384; ssl_prefer_server_ciphers on; @@ -30,6 +30,8 @@ server { client_max_body_size 100M; location / { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5003; include headers.conf; add_header Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; img-src * data:; script-src 'self' 'unsafe-inline'" always; @@ -37,6 +39,8 @@ server { } location /query { + proxy_read_timeout 300s; + proxy_connect_timeout 75s; proxy_pass http://localhost:5103; include graphql.conf; } diff --git a/sr/etc/sr.ht/config.ini b/sr/etc/sr.ht/config.ini index 37a8604..2eb2e0c 100644 --- a/sr/etc/sr.ht/config.ini +++ b/sr/etc/sr.ht/config.ini @@ -55,7 +55,7 @@ private-key=REDACTED origin=http://git.dominic-ricottone.com debug-host=0.0.0.0 debug-port=5001 -#api-origin=http://localhost:5101 +api-origin=http://git.dominic-ricottone.com:5101 # try: meta.localhost/oauth oauth-client-id=CHANGEME oauth-client-secret=CHANGEME @@ -89,7 +89,7 @@ internal-ipnet=127.0.0.0/8,::1/128,192.168.0.0/16,10.0.0.0/8 [meta.sr.ht] origin=http://meta.dominic-ricottone.com welcome-emails=no -#api-origin=http://localhost:5100 +api-origin=http://meta.dominic-ricottone.com:5100 # Debug host debug-host=0.0.0.0 @@ -143,6 +143,7 @@ user-group=srht origin=http://todo.dominic-ricottone.com debug-host=0.0.0.0 debug-port=5003 +api-origin=http://todo.dominic-ricottone.com:5103 # try: meta.localhost/oauth oauth-client-id= oauth-client-secret= @@ -158,7 +159,7 @@ webhooks=redis://localhost:6379/1 [todo.sr.ht::mail] sock=/tmp/todo.sr.ht-lmtp.sock -sock-group=todo +sock-group=postfix posting-domain=lists.tatooine -- 2.45.2