Skip to content

Commit 5b09c33

Browse files
rcstanciurcstanciu
committed
add basic scripts
1 parent 9c141ab commit 5b09c33

File tree

10 files changed

+333
-0
lines changed

10 files changed

+333
-0
lines changed

‎.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.gitignore
2+
docker-compose.yml

‎.env.example‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Base64 encoded Google service account key
2+
GOOGLE_CREDENTIALS=
3+
4+
# Google Cloud Storage bucket name
5+
GOOGLE_BUCKET_NAME=
6+
7+
# Google Cloud Storage backup path
8+
DATABASE_BACKUP_PATH=
9+
10+
# Base64 enoded private key
11+
ENCRYPTION_PRIVATE_KEY=
12+
13+
# Private key passphrase
14+
ENCRYPTION_PASSPHRASE=
15+
16+
# Base64 encoded public key
17+
ENCRYPTION_PUBLIC_KEY=
18+
19+
# Database connection
20+
POSTGRES_HOST=
21+
POSTGRES_PORT=
22+
POSTGRES_USER=
23+
POSTGRES_PASSWORD=
24+
POSTGRES_DB=

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.env

‎Dockerfile‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
FROM google/cloud-sdk:321.0.0-alpine
2+
3+
RUN apk add --no-cache postgresql-client openssl tini
4+
5+
COPY ./scripts /usr/local/bin/scripts
6+
RUN chmod +x /usr/local/bin/scripts/*
7+
RUN mv /usr/local/bin/scripts/* /usr/local/bin \
8+
&& rmdir /usr/local/bin/scripts
9+
10+
COPY ./entrypoint /entrypoint
11+
RUN sed -i 's/\r$//g' /entrypoint
12+
RUN chmod +x /entrypoint
13+
14+
# https://crontab.guru/#*_*/8_*_*_*
15+
RUN echo '* * */8 * * cd /app && bash /app/scripts/backup.sh >> /var/log/pg_backup.log' >> /etc/crontabs/root
16+
17+
WORKDIR /app
18+
ENTRYPOINT ["/entrypoint"]
19+
CMD ["/sbin/tini", "--", "/usr/sbin/crond", "-f", "-l", "8"]

‎docker-compose.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
version: "3.7"
2+
3+
services:
4+
db:
5+
container_name: db
6+
image: postgis/postgis:11-2.5-alpine
7+
networks:
8+
- inner
9+
volumes:
10+
- data:/var/lib/postgresql/data
11+
environment:
12+
- "POSTGRES_USER=${POSTGRES_USER}"
13+
- "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}"
14+
- "POSTGRES_DB=${POSTGRES_DB}"
15+
16+
pg_backup:
17+
container_name: pg_backup
18+
build: .
19+
env_file:
20+
- ./.env
21+
environment:
22+
- "GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS}"
23+
- "GOOGLE_BUCKET_NAME=${GOOGLE_BUCKET_NAME}"
24+
- "ENCRYPTION_PRIVATE_KEY=${ENCRYPTION_PRIVATE_KEY}"
25+
- "ENCRYPTION_PUBLIC_KEY=${ENCRYPTION_PUBLIC_KEY}"
26+
- "ENCRYPTION_PASSPHRASE=${ENCRYPTION_PASSPHRASE}"
27+
- "DATABASE_BACKUP_PATH=${DATABASE_BACKUP_PATH}"
28+
- "POSTGRES_HOST=${POSTGRES_HOST}"
29+
- "POSTGRES_PORT=${POSTGRES_PORT}"
30+
- "POSTGRES_USER=${POSTGRES_USER}"
31+
- "POSTGRES_PASSWORD={POSTGRES_PASSWORD}"
32+
- "POSTGRES_DB=${POSTGRES_DB}"
33+
volumes:
34+
- .:/app
35+
tty: true
36+
networks:
37+
- inner
38+
39+
volumes:
40+
data: {}
41+
42+
networks:
43+
inner:

‎entrypoint‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -o errexit
4+
set -o pipefail
5+
set -o nounset
6+
7+
if [[ -z $GOOGLE_CREDENTIALS ]]; then
8+
echo "Google SDK credentials not set. Ensure GOOGLE_CREDENTIALS contains a valid, B64 encoded, service account key."
9+
exit 1
10+
fi
11+
12+
if [[ -z $GOOGLE_BUCKET_NAME ]]; then
13+
echo "Google Storage bucket name not set. Ensure GOOGLE_BUCKET_NAME contains a valid bucket name."
14+
exit 1
15+
fi
16+
17+
if [[ -z $DATABASE_BACKUP_PATH ]]; then
18+
echo "Database backups path not set. Ensure DATABASE_BACKUP_PATH contains a valid path for database backups."
19+
exit 1
20+
fi
21+
22+
# Setup Google SDK credentials
23+
echo $GOOGLE_CREDENTIALS | base64 -d > /google_credentials.json
24+
export GOOGLE_APPLICATION_CREDENTIALS=/google_credentials.json
25+
gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS
26+
27+
exec "$@"

‎scripts/_sourced/messages.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
# Source: cookiecutter-django template
3+
# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
4+
5+
message_newline() {
6+
echo
7+
}
8+
9+
message_debug()
10+
{
11+
echo -e "DEBUG: ${@}"
12+
}
13+
14+
message_welcome()
15+
{
16+
echo -e "\e[1m${@}\e[0m"
17+
}
18+
19+
message_warning()
20+
{
21+
echo -e "\e[33mWARNING\e[0m: ${@}"
22+
}
23+
24+
message_error()
25+
{
26+
echo -e "\e[31mERROR\e[0m: ${@}"
27+
}
28+
29+
message_info()
30+
{
31+
echo -e "\e[37mINFO\e[0m: ${@}"
32+
}
33+
34+
message_suggestion()
35+
{
36+
echo -e "\e[33mSUGGESTION\e[0m: ${@}"
37+
}
38+
39+
message_success()
40+
{
41+
echo -e "\e[32mSUCCESS\e[0m: ${@}"
42+
}

‎scripts/backup‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env bash
2+
# Based on cookiecutter-django template
3+
# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
4+
5+
set -o errexit
6+
set -o pipefail
7+
set -o nounset
8+
9+
10+
working_dir="$(dirname ${0})"
11+
source "${working_dir}/_sourced/messages.sh"
12+
13+
14+
message_welcome "Backing up the '${POSTGRES_DB}' database..."
15+
16+
17+
if [[ "${POSTGRES_USER}" == "postgres" ]]; then
18+
message_error "Backing up as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
19+
exit 1
20+
fi
21+
22+
export PGHOST="${POSTGRES_HOST}"
23+
export PGPORT="${POSTGRES_PORT}"
24+
export PGUSER="${POSTGRES_USER}"
25+
export PGPASSWORD="${POSTGRES_PASSWORD}"
26+
export PGDATABASE="${POSTGRES_DB}"
27+
28+
random_nonce="$(openssl rand -hex 6)"
29+
timestamp="$(date +'%Y_%m_%dT%H_%M_%S')"
30+
backup_directory="${timestamp}_${random_nonce}"
31+
backup_filename="${backup_directory}/backup_${timestamp}_${random_nonce}.gz"
32+
checksum_filename="${backup_filename}.sha256"
33+
unique_passphrase_file_path="${backup_filename}.pass"
34+
35+
# Create backup directory
36+
mkdir "${backup_directory}"
37+
38+
39+
# Generate a unique passphrase for encrypting the databse dump
40+
openssl rand -hex 128 > "${unique_passphrase_file_path}"
41+
42+
# Dump database
43+
pg_dump -O -x --exclude-table-data vehicles_tripwaypoint -v | gzip > "${backup_filename}"
44+
sha256sum "${backup_filename}" > "${checksum_filename}"
45+
46+
# Encrypt the databse dump using the unique private key
47+
cat "${unique_passphrase_file_path}" | gpg --passphrase-fd 0 --batch --yes --output "${backup_filename}.enc" --symmetric --cipher-algo AES256 "${backup_filename}"
48+
rm "${backup_filename}"
49+
50+
# Encrypt the unique passphrase
51+
public_encryption_key_path="public_encryption_key.key"
52+
echo ${ENCRYPTION_PUBLIC_KEY} | base64 -d > "${public_encryption_key_path}"
53+
private_encryption_key_path="private_encryption_key.key"
54+
echo ${ENCRYPTION_PRIVATE_KEY} | base64 -d > "${private_encryption_key_path}"
55+
encription_passphrase="${ENCRYPTION_PASSPHRASE}"
56+
openssl rsautl -encrypt -inkey "${public_encryption_key_path}" -pubin -in "${unique_passphrase_file_path}" -out "${unique_passphrase_file_path}.enc"
57+
rm -f "${unique_passphrase_file_path}"
58+
59+
# Upload encrypted unique passphrase, encrypted database dump and checksum, zipped
60+
tar c -zvf "${backup_directory}.tar.gz" "${backup_directory}"
61+
set +e
62+
gsutil cp "${backup_directory}.tar.gz" "gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}"; exit_code=$?
63+
set -e
64+
65+
# Cleanup
66+
rm -f "${public_encryption_key_path}"
67+
rm -f "${private_encryption_key_path}"
68+
rm -rf ${backup_directory}
69+
rm -f "${backup_directory}.tar.gz"
70+
71+
if [ $exit_code -ne 0 ]; then
72+
message_error "'${POSTGRES_DB}' database backup '${backup_directory}.tar.gz' has been created but upload failed."
73+
else
74+
message_success "'${POSTGRES_DB}' database backup '${backup_directory}.tar.gz' has been created and uploaded."
75+
fi;

‎scripts/backups‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
# Based on cookiecutter-django template
3+
# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
4+
5+
set -o errexit
6+
set -o pipefail
7+
set -o nounset
8+
9+
10+
working_dir="$(dirname ${0})"
11+
source "${working_dir}/_sourced/messages.sh"
12+
13+
14+
message_welcome "These are the backups you have got:"
15+
backups_path="gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}"
16+
escaped_backups_path="$(echo $backups_path | sed 's/\//\\\//g')"
17+
18+
gsutil ls -lh "${backups_path}" | grep gs:// | sed "s/$escaped_backups_path//g" | sed 's/.tar.gz//g' | sort -r -k 2

‎scripts/restore‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env bash
2+
# Based on cookiecutter-django template
3+
# https://github.com/pydanny/cookiecutter-django/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/compose/production/postgres/maintenance/_sourced/messages.sh
4+
5+
set -o errexit
6+
set -o pipefail
7+
set -o nounset
8+
9+
10+
working_dir="$(dirname ${0})"
11+
source "${working_dir}/_sourced/messages.sh"
12+
13+
14+
if [[ -z ${1+x} ]]; then
15+
message_error "Backup filename is not specified yet it is a required parameter. Make sure you provide one and try again."
16+
exit 1
17+
fi
18+
19+
backup_remote_full_path="gs://${GOOGLE_BUCKET_NAME}${DATABASE_BACKUP_PATH}${1}.tar.gz"
20+
echo $backup_remote_full_path
21+
22+
if ! gsutil -q stat "${backup_remote_full_path}"; then
23+
message_error "No backup with the specified filename found. Check out the 'backups' scripts script output to see if there is one and try again."
24+
exit 1
25+
fi
26+
27+
message_welcome "Restoring the '${POSTGRES_DB}' database from the '${1}' backup..."
28+
29+
if [[ "${POSTGRES_USER}" == "postgres" ]]; then
30+
message_error "Restoring as 'postgres' user is not supported. Assign 'POSTGRES_USER' env with another one and try again."
31+
exit 1
32+
fi
33+
34+
export PGHOST="${POSTGRES_HOST}"
35+
export PGPORT="${POSTGRES_PORT}"
36+
export PGUSER="${POSTGRES_USER}"
37+
export PGPASSWORD="${POSTGRES_PASSWORD}"
38+
export PGDATABASE="${POSTGRES_DB}"
39+
40+
41+
message_info "Downloading backup file..."
42+
gsutil cp "${backup_remote_full_path}" .
43+
44+
message_info "Unzipping backup file..."
45+
tar x -zvf "${1}.tar.gz"
46+
rm -r "${1}.tar.gz"
47+
48+
backup_directory="${1}"
49+
encrypted_backup_filename="${backup_directory}/backup_${1}.gz.enc"
50+
decrypted_backup_filename="${backup_directory}/backup_${1}.gz"
51+
encrypted_pass_filename="${backup_directory}/backup_${1}.gz.pass.enc"
52+
checksum_filename="${backup_directory}/backup_${1}.gz.sha256"
53+
54+
message_info "Decrypting passphrase..."
55+
private_encryption_key_path="private_encryption_key.key"
56+
echo ${ENCRYPTION_PRIVATE_KEY} | base64 -d > "${private_encryption_key_path}"
57+
encription_passphrase="${ENCRYPTION_PASSPHRASE}"
58+
openssl rsautl -decrypt -inkey "${private_encryption_key_path}" -in "${encrypted_pass_filename}" -out "${encrypted_pass_filename}.dec"
59+
60+
message_info "Decrypting backup file..."
61+
cat "${encrypted_pass_filename}.dec" | gpg --passphrase-fd 0 --batch --yes --output "${decrypted_backup_filename}" --decrypt "${encrypted_backup_filename}"
62+
63+
message_info "Validating checksum..."
64+
if ! cat "${checksum_filename}" | sha256sum -c; then
65+
message_error "Backup SHA256 checksum invalid!"
66+
exit 1
67+
fi
68+
69+
message_info "Dropping the database..."
70+
dropdb "${PGDATABASE}"
71+
72+
message_info "Creating a new database..."
73+
createdb --owner="${POSTGRES_USER}" "${PGDATABASE}"
74+
75+
message_info "Applying the backup to the new database..."
76+
gunzip -c "${decrypted_backup_filename}" | psql "${POSTGRES_DB}"
77+
78+
message_info "Cleaning up..."
79+
rm -f "${private_encryption_key_path}"
80+
rm -rf "${backup_directory}"
81+
82+
message_success "The '${POSTGRES_DB}' database has been restored from the '${1}' backup."

0 commit comments

Comments
 (0)