Merge pull request #5 from EbookFoundation/celery-service

Celery as a systemd service

group_vars/batterup/vars.yml

@@ -45,8 +45,6 @@ common_keys:
 host_keys:
   secret_key: '{{ vault_secret_key }}'
   google_books_api_key: "{{ vault_google_books_api_key }}"
-  goodreads_api_key:  "{{ vault_goodreads_api_key }}"
-  goodreads_api_secret:  "{{ vault_goodreads_api_secret }}"
   email_host_user: '{{ vault_email_host_user }}'
   email_host_password:  '{{ vault_email_host_password }}'
   social_auth_twitter_key:  '{{ vault_social_auth_twitter_key }}'

group_vars/dev/vars.yml

@@ -49,8 +49,6 @@ common_keys:
 host_keys:
   secret_key: '{{ vault_secret_key }}'
   google_books_api_key: "{{ vault_google_books_api_key }}"
-  goodreads_api_key:  "{{ vault_goodreads_api_key }}"
-  goodreads_api_secret:  "{{ vault_goodreads_api_secret }}"
   email_host_user: '{{ vault_email_host_user }}'
   email_host_password:  '{{ vault_email_host_password }}'
   social_auth_twitter_key:  '{{ vault_social_auth_twitter_key }}'

group_vars/ondeck/vars.yml

@@ -47,8 +47,6 @@ common_keys:
 host_keys:
   secret_key: '{{ vault_secret_key }}'
   google_books_api_key: "{{ vault_google_books_api_key }}"
-  goodreads_api_key:  "{{ vault_goodreads_api_key }}"
-  goodreads_api_secret:  "{{ vault_goodreads_api_secret }}"
   email_host_user: '{{ vault_email_host_user }}'
   email_host_password:  '{{ vault_email_host_password }}'
   social_auth_twitter_key:  '{{ vault_social_auth_twitter_key }}'

group_vars/production/vars.yml

@@ -47,8 +47,6 @@ common_keys:
 host_keys:
   secret_key: '{{ vault_secret_key }}'
   google_books_api_key: "{{ vault_google_books_api_key }}"
-  goodreads_api_key:  "{{ vault_goodreads_api_key }}"
-  goodreads_api_secret:  "{{ vault_goodreads_api_secret }}"
   email_host_user: '{{ vault_email_host_user }}'
   email_host_password:  '{{ vault_email_host_password }}'
   social_auth_twitter_key:  '{{ vault_social_auth_twitter_key }}'

roles/regluit_common/templates/host.py.j2

@@ -10,9 +10,6 @@ SECRET_KEY = os.environ.get("SECRET_KEY", '0123456789012345678901234567890123456
 # https://code.google.com/apis/console
 GOOGLE_BOOKS_API_KEY = os.environ.get("GOOGLE_BOOKS_API_KEY", "012345678901234567890123456789012345678")
 
-# 
-GOODREADS_API_KEY = os.environ.get("GOODREADS_API_KEY",  "01234567890123456789")
-GOODREADS_API_SECRET = os.environ.get("GOODREADS_API_SECRET",  None) #43 chars
 
 # Amazon SES
 # create with https://console.aws.amazon.com/ses/home?region=us-east-1#smtp-settings:

roles/regluit_prod/files/celerybeat.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Celery Beat Service
+After=network.target
+
+[Service]
+Type=simple
+User=celery
+Group=celery
+EnvironmentFile=/etc/default/celerybeat
+ExecStartPre=-+/bin/mkdir -m 0755 /var/run/celery
+ExecStartPre=+/bin/chown celery:celery /var/run/celery
+WorkingDirectory=/opt/regluit
+ExecStart=/bin/sh -c '"${CELERY_BIN}" -A "${CELERY_APP}" beat  \
+    --pidfile="${CELERYBEAT_PID_FILE}" \
+    --logfile="${CELERYBEAT_LOG_FILE}" --loglevel="${CELERYBEAT_LOG_LEVEL}" \
+    "${CELERYBEAT_OPTS}"'
+
+
+[Install]
+WantedBy=multi-user.target

roles/regluit_prod/files/celeryd.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=Celery Service
+After=network.target
+
+[Service]
+Type=forking
+User=celery
+Group=celery
+EnvironmentFile=/etc/default/celeryd
+ExecStartPre=-+/bin/mkdir -m 0755 /var/run/celery
+ExecStartPre=+/bin/chown celery:celery /var/run/celery
+WorkingDirectory=/opt/regluit
+ExecStart=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi start $CELERYD_NODES \
+    --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \
+    --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS'
+ExecStop=/bin/sh -c '${CELERY_BIN} multi stopwait $CELERYD_NODES \
+    --pidfile=${CELERYD_PID_FILE} --loglevel="${CELERYD_LOG_LEVEL}"'
+ExecReload=/bin/sh -c '${CELERY_BIN} -A $CELERY_APP multi restart $CELERYD_NODES \
+    --pidfile=${CELERYD_PID_FILE} --logfile=${CELERYD_LOG_FILE} \
+    --loglevel="${CELERYD_LOG_LEVEL}" $CELERYD_OPTS'
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

roles/regluit_prod/tasks/celery.yml

@@ -20,31 +20,6 @@
   tags:
     - celery
 
-- name: Create directories for celery
-  become: yes
-  file: 
-    path: "{{ item }}"
-    state: directory
-    owner: celery
-    group: celery
-    mode: 0775
-  with_items:
-    - '/var/log/celery'
-    - '/var/run/celery'
-  tags: 
-    - celery
-
-- name: Copy celery init.d scripts
-  become: yes
-  copy:
-    src: "{{ item }}"
-    dest: "/etc/init.d/{{ item }}"
-    mode: 0755
-  with_items:
-    - 'celeryd'
-    - 'celerybeat'
-  tags: 
-    - celery
 
 - name: Copy celery config files
   become: yes
@@ -58,10 +33,27 @@
   tags: 
     - celery
 
+- name: Copy celery service files
+  become: yes
+  copy:
+    src: "{{ item }}.service"
+    dest: "/etc/systemd/system/{{ item }}.service"
+    mode: 0644
+  with_items:
+    - 'celeryd'
+    - 'celerybeat'
+  tags: 
+    - celery
+
 - name: Start celeryd
   become: yes
-  command: /etc/init.d/celeryd restart
+  ansible.builtin.service:
+    name: celeryd
+    daemon_reload: yes
+    state: reloaded
 
 - name: Start celerybeat
   become: yes
-  command: /etc/init.d/celerybeat restart
+  ansible.builtin.service:
+    name: celerybeat
+    state: restarted

roles/regluit_prod/tasks/certs.yml

@@ -78,6 +78,24 @@
     group: "{{ user_name }}"
     mode: 0755
 
+- name: Create .well-known directory for fresh server
+  become: yes
+  file: 
+    path: "/var/www/html/.well-known"
+    state: directory
+    owner: "{{ user_name }}"
+    group: "{{ user_name }}"
+    mode: 0755
+
+- name: Create acme-challenge directory for fresh server
+  become: yes
+  file: 
+    path: "/var/www/html/.well-known/acme-challenge"
+    state: directory
+    owner: "{{ user_name }}"
+    group: "{{ user_name }}"
+    mode: 0755
+
 - name: debug
   debug:
     msg: "{{ acme_challenge }}"
@@ -94,6 +112,18 @@
     content: "{{ acme_challenge['challenge_data'][alt_server_name]['http-01']['resource_value'] }}"
   when: acme_challenge is changed and alt_server_name != '' and alt_server_name in acme_challenge['challenge_data']
 
+- name: copy main challenge for fresh server
+  copy:
+    dest: /var/www/html/{{ acme_challenge['challenge_data'][server_name]['http-01']['resource'] }}
+    content: "{{ acme_challenge['challenge_data'][server_name]['http-01']['resource_value'] }}"
+  when: acme_challenge is changed and server_name in acme_challenge['challenge_data']
+
+- name: copy alt challenge for fresh server
+  copy:
+    dest: /var/www/html/{{ acme_challenge['challenge_data'][alt_server_name]['http-01']['resource'] }}
+    content: "{{ acme_challenge['challenge_data'][alt_server_name]['http-01']['resource_value'] }}"
+  when: acme_challenge is changed and alt_server_name != '' and alt_server_name in acme_challenge['challenge_data']
+
 - name: Create a challenge for server_name using a account key file.
   acme_certificate:
     account_key_src: private/decrypted/letsencrypt_account.key

roles/regluit_prod/templates/celery/celerybeat.j2

@@ -5,7 +5,7 @@
 CELERYBEAT_CHDIR="{{ project_path }}/"
 
 # Name of the projects settings module.
-export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}"
+DJANGO_SETTINGS_MODULE="{{ django_settings_module }}"
 
 # Path to celery
 CELERY_BIN="{{ project_path }}/{{ virtualenv_name }}/bin/celery"

roles/regluit_prod/templates/celery/celeryd.j2

@@ -9,4 +9,4 @@ CELERYD_GROUP="celery"
 CELERYD_LOG_LEVEL="INFO"
 
 VIRTUALENV_ACTIVATE="{{ project_path }}/{{ virtualenv_name }}/bin/activate"
-export DJANGO_SETTINGS_MODULE="{{ django_settings_module }}"
+DJANGO_SETTINGS_MODULE="{{ django_settings_module }}"