Specify timeout for migrate + allow extending batch jobs to be run (#59)

Author: tobiasmcnulty

.gitignore

@@ -1,3 +1,5 @@
 .idea
 .vscode
 .python-version
+.direnv
+.envrc

CHANGES.rst

@@ -4,6 +4,14 @@ caktus.django-k8s
 Changes
 -------
 
+v1.5.1 on May 24th, 2022
+~~~~~~~~~~~~~~~~~~~~~
+* Rename ``k8s_migration_command`` to ``k8s_migrations_command`` (the old name will continue
+  working for now, but update your projects!)
+* Add ``k8s_migrations_timeout`` variable
+* Support further customizing batch jobs run before and after deploys via the new
+  ``k8s_predeploy_batchjobs`` and ``k8s_postdeploy_batchjobs`` variables
+
 
 v1.5.0 on April 20th, 2022
 ~~~~~~~~~~~~~~~~~~~~~
@@ -20,7 +28,6 @@ v1.5.0 on April 20th, 2022
 * Add support for mounting data volumes via Secrets within containers 
 
 
-
 v1.4.0 on Oct 14, 2021
 ~~~~~~~~~~~~~~~~~~~~~~
 

defaults/main.yml

@@ -172,13 +172,20 @@ k8s_worker_containers:
 k8s_worker_beat_enabled: false
 
 k8s_migrations_enabled: true
-k8s_migration_command:
+k8s_migrations_timeout: 120  # seconds
+k8s_migration_command:  # DEPRECATED: keep name without trailing 's' for backwards compatibility
   - python
   - manage.py
   - migrate
   - --noinput
   - -v
   - "2"
+k8s_migrations_command: "{{ k8s_migration_command }}"
+k8s_migrations_batchjob:
+  job_name: migrate
+  batch_command: "{{ k8s_migrations_command }}"
+  enabled: "{{ k8s_migrations_enabled }}"
+  timeout: "{{ k8s_migrations_timeout }}"
 
 k8s_collectstatic_enabled: true
 k8s_collectstatic_timeout: 120  # seconds
@@ -189,8 +196,21 @@ k8s_collectstatic_command:
   - --noinput
   - -v
   - "2"
-
-k8s_batchjob_state: "{{ (k8s_migrations_enabled or k8s_collectstatic_enabled) | ternary('present', 'absent') }}"
+k8s_collectstatic_batchjob:
+  job_name: collectstatic
+  batch_command: "{{ k8s_collectstatic_command }}"
+  enabled: "{{ k8s_collectstatic_enabled }}"
+  timeout: "{{ k8s_collectstatic_timeout }}"
+
+# k8s_predeploy_batchjobs are one-off jobs that are run *before* every deploy. You can keep
+# the default or override to customize the jobs needed for your project.
+k8s_predeploy_batchjobs:
+  - "{{ k8s_migrations_batchjob }}"
+  - "{{ k8s_collectstatic_batchjob }}"
+# k8s_postdeploy_batchjobs is a hook for running batch jobs *after* every deploy.
+k8s_postdeploy_batchjobs: []
+# k8s_batchjob_state is a hook used to delete job secrets if no batch jobs are enabled
+k8s_batchjob_state: "{{ (k8s_predeploy_batchjobs + k8s_postdeploy_batchjobs) | map(attribute='enabled', default=true) | select | ternary('present', 'absent') }}"
 
 k8s_s3_cluster_name: ""  # name of EKS cluster in AWS
 k8s_s3_region: "us-east-1"

tasks/batchjob_tasks.yml

@@ -0,0 +1,28 @@
+- name: remove any old "{{ batchjob.job_name }}" jobs
+  k8s:
+    api_key: "{{ k8s_auth_api_key }}"
+    host: "{{ k8s_auth_host }}"
+    ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
+    definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
+    state: absent
+    wait: yes
+    validate:
+      fail_on_error: yes
+      strict: yes
+  when: batchjob.enabled | default(true)
+- name: run job "{{ batchjob.job_name }}"
+  k8s:
+    api_key: "{{ k8s_auth_api_key }}"
+    host: "{{ k8s_auth_host }}"
+    ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
+    definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
+    state: present
+    wait: yes
+    wait_condition:
+      type: Complete
+      status: "True"
+    wait_timeout: "{{ batchjob.timeout | default(120) }}"
+    validate:
+      fail_on_error: yes
+      strict: yes
+  when: batchjob.enabled | default(true)

tasks/main.yml

@@ -158,69 +158,11 @@
       type: Opaque
       stringData: "{{ k8s_environment_variables | from_yaml }}"
 
-# Run migrations if wanted
-- when: k8s_migrations_enabled
-  vars:
-    job_name: "migrate"
-    batch_command: "{{ k8s_migration_command }}"
-  block:
-  - name: remove any old migration jobs
-    k8s:
-      api_key: "{{ k8s_auth_api_key }}"
-      host: "{{ k8s_auth_host }}"
-      ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
-      definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
-      state: absent
-      wait: yes
-      validate:
-        fail_on_error: yes
-        strict: yes
-  - name: run migrations
-    k8s:
-      api_key: "{{ k8s_auth_api_key }}"
-      host: "{{ k8s_auth_host }}"
-      ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
-      definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
-      state: present
-      wait: yes
-      wait_condition:
-        type: Complete
-        status: "True"
-      validate:
-        fail_on_error: yes
-        strict: yes
-
-- when: k8s_collectstatic_enabled
-  vars:
-    job_name: "collectstatic"
-    batch_command: "{{ k8s_collectstatic_command }}"
-  block:
-  - name: remove any old collectstatic jobs
-    k8s:
-      api_key: "{{ k8s_auth_api_key }}"
-      host: "{{ k8s_auth_host }}"
-      ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
-      definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
-      state: absent
-      wait: yes
-      validate:
-        fail_on_error: yes
-        strict: yes
-  - name: run collectstatic
-    k8s:
-      api_key: "{{ k8s_auth_api_key }}"
-      host: "{{ k8s_auth_host }}"
-      ca_cert: "{{ k8s_auth_ssl_ca_cert }}"
-      definition: "{{ lookup('template', 'batchjob.yaml.j2') }}"
-      state: present
-      wait: yes
-      wait_condition:
-        type: Complete
-        status: "True"
-      wait_timeout: "{{ k8s_collectstatic_timeout }}"
-      validate:
-        fail_on_error: yes
-        strict: yes
+# Run each pre-deploy batch job in turn, if wanted
+- loop: "{{ k8s_predeploy_batchjobs }}"
+  loop_control:
+    loop_var: batchjob
+  include_tasks: batchjob_tasks.yml
 
 - name: Create/update templates in Kubernetes
   k8s:
@@ -272,3 +214,9 @@
     command: kubectl --namespace "{{ k8s_namespace }}" rollout restart statefulset/celery-beat
     no_log: True
     when: k8s_worker_beat_enabled
+
+# Run each post-deploy batch job in turn, if wanted
+- loop: "{{ k8s_postdeploy_batchjobs }}"
+  loop_control:
+    loop_var: batchjob
+  include_tasks: batchjob_tasks.yml

templates/batchjob.yaml.j2

@@ -2,15 +2,15 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   namespace: "{{ k8s_namespace }}"
-  name: "{{ job_name }}"
+  name: "{{ batchjob.job_name }}"
 spec:
   template:
     spec:
       containers:
       - name: "{{ k8s_container_name }}"
         image: "{{ k8s_container_image }}:{{ k8s_container_image_tag }}"
         imagePullPolicy: "{{ k8s_container_image_pull_policy }}"
-        command: {{ batch_command }}
+        command: {{ batchjob.batch_command }}
         env:
         - name: GET_HOSTS_FROM
           value: dns