Refactor the playbook to support multiple roles per host nicely

Makefile

@@ -1,3 +1,30 @@
 install:
 	ansible-galaxy install geerlingguy.docker geerlingguy.nginx
 	git submodule update --init --recursive
+
+upgrade:
+	ansible-playbook upgrade.yml
+
+mastodon-first-install:
+	ansible-playbook mastodon.yml --tags install,configure --extra-vars "first_install=True"
+
+mastodon:
+	ansible-playbook mastodon.yml --tags install,configure
+mastodon-config:
+	# Used for updating the .env.production settings
+	ansible-playbook mastodon.yml --tags configure
+
+frontend:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'frontend' --extra-vars "install_frontend=True"
+frontend-config:
+	ansible-playbook mastodon.yml --tags configure --limit 'frontend' --extra-vars "install_frontend=True"
+
+webapp:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'webapp' --extra-vars "install_webapp=True"
+webapp-config:
+	ansible-playbook mastodon.yml --tags configure --limit 'webapp' --extra-vars "install_webapp=True"
+
+sidekiq:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'sidekiq' --extra-vars "install_sidekiq=True"
+sidekiq-conifig:
+	ansible-playbook mastodon.yml --tags configure --limit 'sidekiq' --extra-vars "install_sidekiq=True"

README.md

@@ -3,7 +3,6 @@
 Run `make` to install the necessary 3rd party roles, namely:
 - geerlingguy.docker
 - geerlingguy.nginx
-- geerlingguy.certbot
 - oh-my-zsh
 roles
 
@@ -11,7 +10,7 @@ The last role just make the server environments more akin to how I prefer them t
 
 ## Unsupported
 
-This currently doesn't manage anything related to your redis or postgres work.  I'll get to that later, but right now the redis and postgres portions are the oldest parts of my manually installed mastodon instance, and I am not ready to tear that down.
+This currently doesn't manage anything related to your postgres work.  I'll get to that later, but right now the postgres portion is the oldest parts of my manually installed mastodon instance, and I am not ready to tear that down.
 
 ## Configuring your .env.production
 
@@ -55,6 +54,34 @@ ENV_PRODUCTION:
 
 In addition, you'll need to update the `group_vars/all` file and setup the location of your postgres server. This is used by pgbouncer.
 
+## Configuring group_vars
+
+Populate the `group_vars/all` file with something similar to:
+
+``` bash
+MASTODON_VERSION: v4.0.2
+REAL_DB_HOST: hostname-of-your-database
+REAL_DB_PORT: port-of-your-database
+DB_POOL: 25
+```
+
+## Configure your frontends to point to the right webapp
+For each of your frontend hosts, create a file called `host_vars/$HOSTNAME` (where `$HOSTNAME` is the name of each of your frontend machines) and place into it:
+
+``` bash
+webapp_hosts:
+  - HOST_OF_WEBAPP
+  - ANOTHER_HOST_OF_WEBAPP
+```
+for however many webapp (web & streaming) hosts you have.
+
+## Configuring redis
+If you want to install redis on a particular host, create an entry for it under `host_vars/$HOSTNAME` and set the variable:
+
+``` bash
+RUN_REDIS: True
+```
+
 ## Configuring sidekiq workers
 
 It's easy to spin up additional worker capacity.  Edit the `hosts` file, and add your new hostnames under the `[sidekiq]` line, then from the terminal, run `ansible-playbook sidekiq.yml`.  This will configure the host with all of the basic `yttrx` role stuff, including shell and nvim niceties, it will install docker, checkout the mastodon github repo, and configure the `docker-compose.yml` & `docker-compose.override.yml` files.
@@ -65,7 +92,7 @@ The `docker-compose.yml` file is managed by the `mastodon` role, and will simply
 
 Edit the `host_vars/$hostname` (where `$hostname` is a name of a host that is going to run sidekiq) file and configure the `sidekiq` var.  The expected format is:
 
-```
+``` yaml
 sidekiq:
   - name: ingress-and-stuff
   - q: [ 'ingress', 'default' ]

hosts

@@ -1,10 +1,8 @@
 [yttrx]
-grafana.yttrx.com ansible_user=root
+tusky.masto.yttrx.com
-tusky.masto.yttrx.com ansible_user=root
-mail.yttrx.com ansible_user=root
 wooly.masto.yttrx.com
 
-[sidekiq]
+[mastodon]
 wooly.masto.yttrx.com
 
 [frontend]
@@ -13,11 +11,8 @@ wooly.masto.yttrx.com
 [webapp]
 wooly.masto.yttrx.com
 
+[sidekiq]
+wooly.masto.yttrx.com
+
 [yttrx:vars]
 ansible_python_interpreter=/usr/bin/python3
-
-[mta]
-mail.yttrx.com ansible_user=root
-
-[mta:vars]
-ansible_python_interpreter=/usr/bin/python3

mastodon.yml

@@ -0,0 +1,30 @@
+---
+- name: Configure yttrx mastodon server
+  hosts: [mastodon]
+  vars:
+    first_install: False
+    install_sidekiq: False
+    install_webapp: False
+    install_frontend: False
+
+  roles:
+    - when: first_install | bool
+      role: yttrx
+      tags:
+        - firstinstall
+
+    - role: mastodon
+    
+    - when: install_sidekiq| bool
+      role: sidekiq
+    - when: install_webapp | bool
+      role: webapp
+    - when: install_frontend | bool
+      role: frontend
+
+  handlers:
+  - name: Re-up docker containers
+    command: /usr/local/bin/docker-compose.sh
+    args:
+      chdir: /home/mastodon/live
+    listen: "docker-compose up"

roles/frontend/tasks/main.yml

@@ -7,41 +7,55 @@
     state: directory
     owner: www-data
     group: www-data
+  tags:
+    - install
 
-- name: Install nginx site cdn
+- name: Configure nginx site cdn
   template:
     src: cdn.j2
     dest: /etc/nginx/sites-available/cdn
   notify: restart nginx
+  tags:
+    - configure
 
-- name: Install nginx mastodon config
+- name: Configure nginx mastodon config
   template:
     src: mastodon.j2
     dest: /etc/nginx/sites-available/mastodon
   notify: restart nginx
+  tags:
+    - configure
 
 - name: Enable cdn
   file:
     src: /etc/nginx/sites-available/cdn
     dest: /etc/nginx/sites-enabled/cdn
     state: link
+  tags:
+    - install
 
 - name: Enable mastodon
   file:
     src: /etc/nginx/sites-available/mastodon
     dest: /etc/nginx/sites-enabled/mastodon
     state: link
+  tags:
+    - install
 
 - name: Disable default
   file:
     path: /etc/nginx/sites-enabled/default
     state: absent
+  tags:
+    - install
 
 - name: Copy dmca
   template:
     src: dmca
     dest: /etc/nginx/dmca
   notify: restart nginx
+  tags:
+    - install
 
 - name: Prep SSL directory
   file:
@@ -49,30 +63,41 @@
     state: directory
     owner: www-data
     group: www-data
+  tags:
+    - install
 
 - name: Install mastodon domain SSL certificate
   template:
     src: domain.fullchain.pem
     dest: /etc/nginx/ssl/domain.fullchain.pem
+  tags:
+    - install
 
 - name: Install mastodon domain ssl key
   template:
     src: domain.privkey.pem
     dest: /etc/nginx/ssl/domain.privkey.pem
+  tags:
+    - install
 
 - name: Install mastodon CDN SSL certificate
   template:
     src: cdn.fullchain.pem
     dest: /etc/nginx/ssl/cdn.fullchain.pem
+  tags:
+    - install
 
 - name: Install mastodon CDN ssl key
   template:
     src: cdn.privkey.pem
     dest: /etc/nginx/ssl/cdn.privkey.pem
+  tags:
+    - install
 
 - name: Copy cloudflare
   template:
     src: cloudflare
     dest: /etc/nginx/cloudflare
   notify: restart nginx
-
+  tags:
+    - install

roles/mastodon/files/pull-images.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose $(ls /home/mastodon/live/docker-compose.*yml -1 | sort -r | awk '{printf "-f %s ", $1}') pull

roles/mastodon/meta/main.yml

@@ -1,4 +1 @@
 ---
-dependencies:
-  - role: geerlingguy.docker
-  - role: yttrx

roles/mastodon/tasks/main.yml

@@ -5,6 +5,8 @@
     name: mastodon
     shell: /bin/bash
     group: docker
+  tags:
+    - install
 
 - name: Check if mastodon repo checked out
   stat: path=/home/mastodon/live
@@ -14,19 +16,25 @@
   when: not p.stat.exists
 
 - name: Install statsd mapper
-  template:
+  copy:
-    src: statsd-mapping.yaml.j2
+    src: statsd-mapping.yaml
     dest: /root/statsd-mapping.yml
+  tags:
+    - install
 
 - name: Install .env.production file
   template:
     src: env.production.j2
     dest: /home/mastodon/live/.env.production
+  tags:
+    - configure
 
 - name: Install docker-compose.sh file
-  template:
+  copy:
-    src: docker-compose.sh.j2
+    src: docker-compose.sh
     dest: /usr/local/bin/docker-compose.sh
+  tags:
+    - install
 
 - name: Set permissions of docker-compose.sh
   file:
@@ -37,12 +45,16 @@
   template:
     src: docker-compose.yml.j2
     dest: /home/mastodon/live/docker-compose.yml
+  tags:
+    - configure
   notify: "docker-compose up"
 
 - name: Install feed rebuild script
   template:
     src: feeds-build.sh.j2
     dest: /usr/local/bin/feeds-build.sh 
+  tags:
+    - install
 
 - name: Set permission of feeds-build.sh
   file:
@@ -54,4 +66,4 @@
   when: RUN_REDIS is defined and RUN_REDIS
 
 - import_tasks: maintenance.yml
-  when: ansible_hostname == MAINTENANCE_HOST
+  when: RUN_CRONS is defined and RUN_CRONS

roles/mastodon/tasks/maintenance.yml

@@ -1,8 +1,19 @@
 ---
 - name: Install storage trimmer
-  template:
+  copy:
-    src: trim_storage.sh.j2
+    src: trim_storage.sh
     dest: /usr/local/bin/trim_storage.sh
+  tags:
+    - install
+  register: trim_storage
+
+- name: Set permissions on trim_storage.sh
+  file:
+    path: /usr/local/bin/trim_storage.sh
+    mode: '0755'
+  tags:
+    - install
+  when: trim_storage.changed
 
 - name: Create cronjob for trimming storage
   ansible.builtin.cron:
@@ -10,3 +21,21 @@
     job: /usr/local/bin/trim_storage.sh
     minute: "0"
     hour: "*/6"
+  tags:
+    - configure
+
+- name: Install image puller
+  copy:
+    src: pull-images.sh
+    dest: /usr/local/bin/pull-images.sh
+  tags:
+    - install
+  register: pull_images
+
+- name: Set permissions on pull-images.sh
+  file:
+    path: /usr/local/bin/pull-images.sh
+    mode: '0755'
+  tags:
+    - install
+  when: pull_images.changed

roles/mastodon/tasks/redis.yml

@@ -2,17 +2,26 @@
   sysctl:
     name: vm.overcommit_memory
     value: '1'
+  tags:
+    - redis
 
 - name: Install system redis
   package:
     name: redis-server
     state: present
+  tags:
+    - install
+    - redis
 
 - name: Set redis.conf
   copy:
     dest: /etc/redis/redis.conf
     src: redis-6.conf
   register: service_conf
+  tags:
+    - install
+    - configure
+    - redis
 
 - name: Restart redis
   service:

roles/mastodon/tasks/repo.yml

@@ -4,4 +4,3 @@
     repo: https://github.com/mastodon/mastodon.git
     dest: /home/mastodon/live
     version: "{{ MASTODON_VERSION }}"
-

roles/sidekiq/meta/main.yml

@@ -1,3 +1 @@
 ---
-dependencies:
-  - role: mastodon

roles/sidekiq/tasks/main.yml

@@ -1,6 +1,9 @@
 ---
   - name: Configure docker-compose.override.yml
     template:
-      src: docker-compose.override.yml.j2
+      src: docker-compose.sidekiq.yml.j2
       dest: /home/mastodon/live/docker-compose.sidekiq.yml
     notify: "docker-compose up"
+    tags:
+      - configure
+      - sidekiq

roles/webapp/meta/main.yml

@@ -1,4 +1 @@
 ---
-dependencies:
-  - role: geerlingguy.nginx
-  - role: mastodon

roles/webapp/tasks/main.yml

@@ -4,3 +4,6 @@
       src: docker-compose.override.yml.j2
       dest: /home/mastodon/live/docker-compose.webapp.yml
     notify: "docker-compose up"
+    tags:
+      - configure
+      - webapp

sidekiq.yml

@@ -1,12 +0,0 @@
----
-- name: Configure yttrx sidekiq environment
-  hosts: [sidekiq]
-  roles:
-    - role: sidekiq
-
-  handlers:
-  - name: Re-up docker containers
-    command: /usr/local/bin/docker-compose.sh
-    args:
-      chdir: /home/mastodon/live
-    listen: "docker-compose up"