From 4d0b70bbe1cdc03f91424c85787d9aba17668b61 Mon Sep 17 00:00:00 2001
From: Waffles <waffles@yttrx.com>
Date: Mon, 9 Jan 2023 02:39:02 -0800
Subject: [PATCH] Refactor the playbook to support multiple roles per host
 nicely

---
 Makefile                                      | 27 +++++++++++++++
 README.md                                     | 33 +++++++++++++++++--
 hosts                                         | 15 +++------
 mastodon.yml                                  | 30 +++++++++++++++++
 roles/frontend/tasks/main.yml                 | 31 +++++++++++++++--
 .../docker-compose.sh}                        |  0
 roles/mastodon/files/pull-images.sh           |  2 ++
 .../statsd-mapping.yaml}                      |  0
 .../trim_storage.sh}                          |  0
 roles/mastodon/meta/main.yml                  |  3 --
 roles/mastodon/tasks/main.yml                 | 22 ++++++++++---
 roles/mastodon/tasks/maintenance.yml          | 33 +++++++++++++++++--
 roles/mastodon/tasks/redis.yml                |  9 +++++
 roles/mastodon/tasks/repo.yml                 |  1 -
 roles/sidekiq/meta/main.yml                   |  2 --
 roles/sidekiq/tasks/main.yml                  |  5 ++-
 ...e.yml.j2 => docker-compose.sidekiq.yml.j2} |  0
 roles/webapp/meta/main.yml                    |  3 --
 roles/webapp/tasks/main.yml                   |  3 ++
 sidekiq.yml                                   | 12 -------
 20 files changed, 186 insertions(+), 45 deletions(-)
 create mode 100644 mastodon.yml
 rename roles/mastodon/{templates/docker-compose.sh.j2 => files/docker-compose.sh} (100%)
 create mode 100644 roles/mastodon/files/pull-images.sh
 rename roles/mastodon/{templates/statsd-mapping.yaml.j2 => files/statsd-mapping.yaml} (100%)
 rename roles/mastodon/{templates/trim_storage.sh.j2 => files/trim_storage.sh} (100%)
 rename roles/sidekiq/templates/{docker-compose.override.yml.j2 => docker-compose.sidekiq.yml.j2} (100%)
 delete mode 100644 sidekiq.yml

diff --git a/Makefile b/Makefile
index 299e94b..31f52dc 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,30 @@
 install:
 	ansible-galaxy install geerlingguy.docker geerlingguy.nginx
 	git submodule update --init --recursive
+
+upgrade:
+	ansible-playbook upgrade.yml
+
+mastodon-first-install:
+	ansible-playbook mastodon.yml --tags install,configure --extra-vars "first_install=True"
+
+mastodon:
+	ansible-playbook mastodon.yml --tags install,configure
+mastodon-config:
+	# Used for updating the .env.production settings
+	ansible-playbook mastodon.yml --tags configure
+
+frontend:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'frontend' --extra-vars "install_frontend=True"
+frontend-config:
+	ansible-playbook mastodon.yml --tags configure --limit 'frontend' --extra-vars "install_frontend=True"
+
+webapp:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'webapp' --extra-vars "install_webapp=True"
+webapp-config:
+	ansible-playbook mastodon.yml --tags configure --limit 'webapp' --extra-vars "install_webapp=True"
+
+sidekiq:
+	ansible-playbook mastodon.yml --tags install,configure --limit 'sidekiq' --extra-vars "install_sidekiq=True"
+sidekiq-conifig:
+	ansible-playbook mastodon.yml --tags configure --limit 'sidekiq' --extra-vars "install_sidekiq=True"
diff --git a/README.md b/README.md
index 2088b41..759b77f 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,6 @@
 Run `make` to install the necessary 3rd party roles, namely:
 - geerlingguy.docker
 - geerlingguy.nginx
-- geerlingguy.certbot
 - oh-my-zsh
 roles
 
@@ -11,7 +10,7 @@ The last role just make the server environments more akin to how I prefer them t
 
 ## Unsupported
 
-This currently doesn't manage anything related to your redis or postgres work.  I'll get to that later, but right now the redis and postgres portions are the oldest parts of my manually installed mastodon instance, and I am not ready to tear that down.
+This currently doesn't manage anything related to your postgres work.  I'll get to that later, but right now the postgres portion is the oldest parts of my manually installed mastodon instance, and I am not ready to tear that down.
 
 ## Configuring your .env.production
 
@@ -55,6 +54,34 @@ ENV_PRODUCTION:
 
 In addition, you'll need to update the `group_vars/all` file and setup the location of your postgres server. This is used by pgbouncer.
 
+## Configuring group_vars
+
+Populate the `group_vars/all` file with something similar to:
+
+``` bash
+MASTODON_VERSION: v4.0.2
+REAL_DB_HOST: hostname-of-your-database
+REAL_DB_PORT: port-of-your-database
+DB_POOL: 25
+```
+
+## Configure your frontends to point to the right webapp
+For each of your frontend hosts, create a file called `host_vars/$HOSTNAME` (where `$HOSTNAME` is the name of each of your frontend machines) and place into it:
+
+``` bash
+webapp_hosts:
+  - HOST_OF_WEBAPP
+  - ANOTHER_HOST_OF_WEBAPP
+```
+for however many webapp (web & streaming) hosts you have.
+
+## Configuring redis
+If you want to install redis on a particular host, create an entry for it under `host_vars/$HOSTNAME` and set the variable:
+
+``` bash
+RUN_REDIS: True
+```
+
 ## Configuring sidekiq workers
 
 It's easy to spin up additional worker capacity.  Edit the `hosts` file, and add your new hostnames under the `[sidekiq]` line, then from the terminal, run `ansible-playbook sidekiq.yml`.  This will configure the host with all of the basic `yttrx` role stuff, including shell and nvim niceties, it will install docker, checkout the mastodon github repo, and configure the `docker-compose.yml` & `docker-compose.override.yml` files.
@@ -65,7 +92,7 @@ The `docker-compose.yml` file is managed by the `mastodon` role, and will simply
 
 Edit the `host_vars/$hostname` (where `$hostname` is a name of a host that is going to run sidekiq) file and configure the `sidekiq` var.  The expected format is:
 
-```
+``` yaml
 sidekiq:
   - name: ingress-and-stuff
   - q: [ 'ingress', 'default' ]
diff --git a/hosts b/hosts
index 5620be8..180c132 100644
--- a/hosts
+++ b/hosts
@@ -1,10 +1,8 @@
 [yttrx]
-grafana.yttrx.com ansible_user=root
-tusky.masto.yttrx.com ansible_user=root
-mail.yttrx.com ansible_user=root
+tusky.masto.yttrx.com
 wooly.masto.yttrx.com
 
-[sidekiq]
+[mastodon]
 wooly.masto.yttrx.com
 
 [frontend]
@@ -13,11 +11,8 @@ wooly.masto.yttrx.com
 [webapp]
 wooly.masto.yttrx.com
 
+[sidekiq]
+wooly.masto.yttrx.com
+
 [yttrx:vars]
 ansible_python_interpreter=/usr/bin/python3
-
-[mta]
-mail.yttrx.com ansible_user=root
-
-[mta:vars]
-ansible_python_interpreter=/usr/bin/python3
diff --git a/mastodon.yml b/mastodon.yml
new file mode 100644
index 0000000..2d7ae5f
--- /dev/null
+++ b/mastodon.yml
@@ -0,0 +1,30 @@
+---
+- name: Configure yttrx mastodon server
+  hosts: [mastodon]
+  vars:
+    first_install: False
+    install_sidekiq: False
+    install_webapp: False
+    install_frontend: False
+
+  roles:
+    - when: first_install | bool
+      role: yttrx
+      tags:
+        - firstinstall
+
+    - role: mastodon
+    
+    - when: install_sidekiq| bool
+      role: sidekiq
+    - when: install_webapp | bool
+      role: webapp
+    - when: install_frontend | bool
+      role: frontend
+
+  handlers:
+  - name: Re-up docker containers
+    command: /usr/local/bin/docker-compose.sh
+    args:
+      chdir: /home/mastodon/live
+    listen: "docker-compose up"
diff --git a/roles/frontend/tasks/main.yml b/roles/frontend/tasks/main.yml
index 50c5551..316fe16 100644
--- a/roles/frontend/tasks/main.yml
+++ b/roles/frontend/tasks/main.yml
@@ -7,41 +7,55 @@
     state: directory
     owner: www-data
     group: www-data
+  tags:
+    - install
 
-- name: Install nginx site cdn
+- name: Configure nginx site cdn
   template:
     src: cdn.j2
     dest: /etc/nginx/sites-available/cdn
   notify: restart nginx
+  tags:
+    - configure
 
-- name: Install nginx mastodon config
+- name: Configure nginx mastodon config
   template:
     src: mastodon.j2
     dest: /etc/nginx/sites-available/mastodon
   notify: restart nginx
+  tags:
+    - configure
 
 - name: Enable cdn
   file:
     src: /etc/nginx/sites-available/cdn
     dest: /etc/nginx/sites-enabled/cdn
     state: link
+  tags:
+    - install
 
 - name: Enable mastodon
   file:
     src: /etc/nginx/sites-available/mastodon
     dest: /etc/nginx/sites-enabled/mastodon
     state: link
+  tags:
+    - install
 
 - name: Disable default
   file:
     path: /etc/nginx/sites-enabled/default
     state: absent
+  tags:
+    - install
 
 - name: Copy dmca
   template:
     src: dmca
     dest: /etc/nginx/dmca
   notify: restart nginx
+  tags:
+    - install
 
 - name: Prep SSL directory
   file:
@@ -49,30 +63,41 @@
     state: directory
     owner: www-data
     group: www-data
+  tags:
+    - install
 
 - name: Install mastodon domain SSL certificate
   template:
     src: domain.fullchain.pem
     dest: /etc/nginx/ssl/domain.fullchain.pem
+  tags:
+    - install
 
 - name: Install mastodon domain ssl key
   template:
     src: domain.privkey.pem
     dest: /etc/nginx/ssl/domain.privkey.pem
+  tags:
+    - install
 
 - name: Install mastodon CDN SSL certificate
   template:
     src: cdn.fullchain.pem
     dest: /etc/nginx/ssl/cdn.fullchain.pem
+  tags:
+    - install
 
 - name: Install mastodon CDN ssl key
   template:
     src: cdn.privkey.pem
     dest: /etc/nginx/ssl/cdn.privkey.pem
+  tags:
+    - install
 
 - name: Copy cloudflare
   template:
     src: cloudflare
     dest: /etc/nginx/cloudflare
   notify: restart nginx
-
+  tags:
+    - install
diff --git a/roles/mastodon/templates/docker-compose.sh.j2 b/roles/mastodon/files/docker-compose.sh
similarity index 100%
rename from roles/mastodon/templates/docker-compose.sh.j2
rename to roles/mastodon/files/docker-compose.sh
diff --git a/roles/mastodon/files/pull-images.sh b/roles/mastodon/files/pull-images.sh
new file mode 100644
index 0000000..4fdb0ba
--- /dev/null
+++ b/roles/mastodon/files/pull-images.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose $(ls /home/mastodon/live/docker-compose.*yml -1 | sort -r | awk '{printf "-f %s ", $1}') pull
diff --git a/roles/mastodon/templates/statsd-mapping.yaml.j2 b/roles/mastodon/files/statsd-mapping.yaml
similarity index 100%
rename from roles/mastodon/templates/statsd-mapping.yaml.j2
rename to roles/mastodon/files/statsd-mapping.yaml
diff --git a/roles/mastodon/templates/trim_storage.sh.j2 b/roles/mastodon/files/trim_storage.sh
similarity index 100%
rename from roles/mastodon/templates/trim_storage.sh.j2
rename to roles/mastodon/files/trim_storage.sh
diff --git a/roles/mastodon/meta/main.yml b/roles/mastodon/meta/main.yml
index c0ff46d..ed97d53 100644
--- a/roles/mastodon/meta/main.yml
+++ b/roles/mastodon/meta/main.yml
@@ -1,4 +1 @@
 ---
-dependencies:
-  - role: geerlingguy.docker
-  - role: yttrx
diff --git a/roles/mastodon/tasks/main.yml b/roles/mastodon/tasks/main.yml
index 3352e23..9e992ee 100644
--- a/roles/mastodon/tasks/main.yml
+++ b/roles/mastodon/tasks/main.yml
@@ -5,6 +5,8 @@
     name: mastodon
     shell: /bin/bash
     group: docker
+  tags:
+    - install
 
 - name: Check if mastodon repo checked out
   stat: path=/home/mastodon/live
@@ -14,19 +16,25 @@
   when: not p.stat.exists
 
 - name: Install statsd mapper
-  template:
-    src: statsd-mapping.yaml.j2
+  copy:
+    src: statsd-mapping.yaml
     dest: /root/statsd-mapping.yml
+  tags:
+    - install
 
 - name: Install .env.production file
   template:
     src: env.production.j2
     dest: /home/mastodon/live/.env.production
+  tags:
+    - configure
 
 - name: Install docker-compose.sh file
-  template:
-    src: docker-compose.sh.j2
+  copy:
+    src: docker-compose.sh
     dest: /usr/local/bin/docker-compose.sh
+  tags:
+    - install
 
 - name: Set permissions of docker-compose.sh
   file:
@@ -37,12 +45,16 @@
   template:
     src: docker-compose.yml.j2
     dest: /home/mastodon/live/docker-compose.yml
+  tags:
+    - configure
   notify: "docker-compose up"
 
 - name: Install feed rebuild script
   template:
     src: feeds-build.sh.j2
     dest: /usr/local/bin/feeds-build.sh 
+  tags:
+    - install
 
 - name: Set permission of feeds-build.sh
   file:
@@ -54,4 +66,4 @@
   when: RUN_REDIS is defined and RUN_REDIS
 
 - import_tasks: maintenance.yml
-  when: ansible_hostname == MAINTENANCE_HOST
+  when: RUN_CRONS is defined and RUN_CRONS
diff --git a/roles/mastodon/tasks/maintenance.yml b/roles/mastodon/tasks/maintenance.yml
index 5bca710..7d71642 100644
--- a/roles/mastodon/tasks/maintenance.yml
+++ b/roles/mastodon/tasks/maintenance.yml
@@ -1,8 +1,19 @@
 ---
 - name: Install storage trimmer
-  template:
-    src: trim_storage.sh.j2
+  copy:
+    src: trim_storage.sh
     dest: /usr/local/bin/trim_storage.sh
+  tags:
+    - install
+  register: trim_storage
+
+- name: Set permissions on trim_storage.sh
+  file:
+    path: /usr/local/bin/trim_storage.sh
+    mode: '0755'
+  tags:
+    - install
+  when: trim_storage.changed
 
 - name: Create cronjob for trimming storage
   ansible.builtin.cron:
@@ -10,3 +21,21 @@
     job: /usr/local/bin/trim_storage.sh
     minute: "0"
     hour: "*/6"
+  tags:
+    - configure
+
+- name: Install image puller
+  copy:
+    src: pull-images.sh
+    dest: /usr/local/bin/pull-images.sh
+  tags:
+    - install
+  register: pull_images
+
+- name: Set permissions on pull-images.sh
+  file:
+    path: /usr/local/bin/pull-images.sh
+    mode: '0755'
+  tags:
+    - install
+  when: pull_images.changed
diff --git a/roles/mastodon/tasks/redis.yml b/roles/mastodon/tasks/redis.yml
index abf40c6..f7e28d7 100644
--- a/roles/mastodon/tasks/redis.yml
+++ b/roles/mastodon/tasks/redis.yml
@@ -2,17 +2,26 @@
   sysctl:
     name: vm.overcommit_memory
     value: '1'
+  tags:
+    - redis
 
 - name: Install system redis
   package:
     name: redis-server
     state: present
+  tags:
+    - install
+    - redis
 
 - name: Set redis.conf
   copy:
     dest: /etc/redis/redis.conf
     src: redis-6.conf
   register: service_conf
+  tags:
+    - install
+    - configure
+    - redis
 
 - name: Restart redis
   service:
diff --git a/roles/mastodon/tasks/repo.yml b/roles/mastodon/tasks/repo.yml
index 0178834..06391b9 100644
--- a/roles/mastodon/tasks/repo.yml
+++ b/roles/mastodon/tasks/repo.yml
@@ -4,4 +4,3 @@
     repo: https://github.com/mastodon/mastodon.git
     dest: /home/mastodon/live
     version: "{{ MASTODON_VERSION }}"
-
diff --git a/roles/sidekiq/meta/main.yml b/roles/sidekiq/meta/main.yml
index 6786fd2..ed97d53 100644
--- a/roles/sidekiq/meta/main.yml
+++ b/roles/sidekiq/meta/main.yml
@@ -1,3 +1 @@
 ---
-dependencies:
-  - role: mastodon
diff --git a/roles/sidekiq/tasks/main.yml b/roles/sidekiq/tasks/main.yml
index 4cceefa..d0c05d3 100644
--- a/roles/sidekiq/tasks/main.yml
+++ b/roles/sidekiq/tasks/main.yml
@@ -1,6 +1,9 @@
 ---
   - name: Configure docker-compose.override.yml
     template:
-      src: docker-compose.override.yml.j2
+      src: docker-compose.sidekiq.yml.j2
       dest: /home/mastodon/live/docker-compose.sidekiq.yml
     notify: "docker-compose up"
+    tags:
+      - configure
+      - sidekiq
diff --git a/roles/sidekiq/templates/docker-compose.override.yml.j2 b/roles/sidekiq/templates/docker-compose.sidekiq.yml.j2
similarity index 100%
rename from roles/sidekiq/templates/docker-compose.override.yml.j2
rename to roles/sidekiq/templates/docker-compose.sidekiq.yml.j2
diff --git a/roles/webapp/meta/main.yml b/roles/webapp/meta/main.yml
index 2057a43..ed97d53 100644
--- a/roles/webapp/meta/main.yml
+++ b/roles/webapp/meta/main.yml
@@ -1,4 +1 @@
 ---
-dependencies:
-  - role: geerlingguy.nginx
-  - role: mastodon
diff --git a/roles/webapp/tasks/main.yml b/roles/webapp/tasks/main.yml
index 97cfbd4..4a445e5 100644
--- a/roles/webapp/tasks/main.yml
+++ b/roles/webapp/tasks/main.yml
@@ -4,3 +4,6 @@
       src: docker-compose.override.yml.j2
       dest: /home/mastodon/live/docker-compose.webapp.yml
     notify: "docker-compose up"
+    tags:
+      - configure
+      - webapp
diff --git a/sidekiq.yml b/sidekiq.yml
deleted file mode 100644
index 1ada40b..0000000
--- a/sidekiq.yml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-- name: Configure yttrx sidekiq environment
-  hosts: [sidekiq]
-  roles:
-    - role: sidekiq
-
-  handlers:
-  - name: Re-up docker containers
-    command: /usr/local/bin/docker-compose.sh
-    args:
-      chdir: /home/mastodon/live
-    listen: "docker-compose up"