diff --git a/MANAGE/Dockerfile b/MANAGE/Dockerfile index 4185c3e..b802a07 100644 --- a/MANAGE/Dockerfile +++ b/MANAGE/Dockerfile @@ -1,9 +1,17 @@ -FROM node:7.10.1-slim +FROM python:3.6-slim + +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.name="symptomsurvey_backed/DB" +LABEL org.label-schema.description="Scripts for setting up the database used by the other sections of project" +LABEL org.label-schema.url="http://www.codeforpdx.org/projects/2" +LABEL org.label-schema.vcs-url="https://github.com/codeforpdx/symptomsurvey_backend" WORKDIR /usr/src/app -#do this BEFORE copying the rest, that way only changes to package.json will cause npm to execute -COPY ./MANAGE/package.json ./package.json -RUN npm install +RUN pip install --upgrade pip +RUN pip3 install pymongo -COPY ./MANAGE ./SHARED ./ +COPY ./MANAGE /MANAGE +COPY ./SHARED/modules/common.py /MANAGE/common.py +COPY ./SHARED/json /SHARED/json +CMD ["python", "/MANAGE/setup_mongo.py"] diff --git a/MANAGE/migrate-mongo-config.js b/MANAGE/migrate-mongo-config.js deleted file mode 100644 index fc7215e..0000000 --- a/MANAGE/migrate-mongo-config.js +++ /dev/null @@ -1,23 +0,0 @@ -const {database_name} = require('./constants'); - -const mongoHost = process.env.MONGO_HOST || 'mongo'; -const mongoPort = process.env.MONGO_PORT || 27017; - -module.exports = { - mongodb: { - // TODO: set this url in an environment variable so that it is configurable. - url: `mongodb://${mongoHost}:${mongoPort}`, - - databaseName: database_name, - - options: { - useNewUrlParser: true // removes a deprecation warning when connecting - } - }, - - // The migrations dir, can be an relative or absolute path. Only edit this when really necessary. - migrationsDir: "migrations", - - // The mongodb collection where the applied changes are stored. Only edit this when really necessary. - changelogCollectionName: "changelog" -}; diff --git a/MANAGE/migrations/20181211035401-populate_users.js b/MANAGE/migrations/20181211035401-populate_users.js deleted file mode 100644 index 8ef646c..0000000 --- a/MANAGE/migrations/20181211035401-populate_users.js +++ /dev/null @@ -1,51 +0,0 @@ -const {pbkdf2} = require('crypto'); -const uuidv4 = require('uuid/v4'); -const pick = require('lodash/pick'); -const {salt: {iterations, key_length, algorithm}} = require('../constants'); -const userSeeds = require('../seeds/users'); - -/** - * makeHash - * @param {string} password - The user's actual password. - * @param {string} salt - The randomly generated salt. - * @returns {Promise} A promise that resolves to a hex string of the password hash - */ -const makeHash = (password, salt) => new Promise((resolve, reject) => pbkdf2(password, salt, Number(iterations), Number(key_length), algorithm, (err, result) => { - if (err) { - reject(err); - return; - } - resolve(result.toString('hex')); -})); - -const hashUserPassword = user => { - const salt = uuidv4(); - return makeHash(user.password, salt) - .then(hash => { - return Object.assign( - {}, - pick(user, ['profile', 'role']), - {password: { - salt, - hash - }} - ) - }) -} - -module.exports = { - up(db) { - // Generate both password hashes and create the users collection. - return db.createCollection('users') - .then(() => Promise.all(userSeeds.map(hashUserPassword))) - .then(users => { - // Insert both users with the appropriate salt/passwordHash values. - return db.collection('users').insertMany(users); - }); - }, - - down(db) { - // Running this down script will destroy all data in the users table. - return db.collection('users').drop(); - } -}; diff --git a/MANAGE/migrations/20181217051047-permissions.js b/MANAGE/migrations/20181217051047-permissions.js deleted file mode 100644 index 7b57aeb..0000000 --- a/MANAGE/migrations/20181217051047-permissions.js +++ /dev/null @@ -1,12 +0,0 @@ -const rolesSeeds = require('../seeds/roles'); - -module.exports = { - up(db) { - return db.createCollection('roles') - .then(() => db.collection('roles').insertMany(rolesSeeds)); - }, - - down(db) { - return db.collection('roles').drop(); - } -}; diff --git a/MANAGE/migrations/20190321035400-tweets.js b/MANAGE/migrations/20190321035400-tweets.js deleted file mode 100644 index 8b425b2..0000000 --- a/MANAGE/migrations/20190321035400-tweets.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = { - up(db) { - return db - .createCollection('tweets') - .then(() => db.collection('tweets').createIndex({text: 'text'})) - .then(() => db.collection('tweets').createIndex({geo: '2dsphere'})); - }, - - down(db) { - return db.collection('tweets').drop(); - } -}; diff --git a/MANAGE/package.json b/MANAGE/package.json deleted file mode 100644 index 4e7f45e..0000000 --- a/MANAGE/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "symptomsurvey_backend", - "version": "0.0.0", - "description": "A node module that runs database migrations for the Symptom Survey Code for PDX project.", - "main": "index.js", - "scripts": { - "create": "migrate-mongo create", - "down": "migrate-mongo down", - "up": "migrate-mongo up" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/CodeForPortland/symptomsurvey_backend.git" - }, - "author": "Code for PDX", - "license": "ISC", - "bugs": { - "url": "https://github.com/CodeForPortland/symptomsurvey_backend/issues" - }, - "homepage": "https://github.com/CodeForPortland/symptomsurvey_backend#readme", - "dependencies": { - "lodash": "^4.17.11", - "migrate-mongo": "4.1.2", - "uuid": "^3.3.2" - } -} diff --git a/MANAGE/setup_mongo.py b/MANAGE/setup_mongo.py new file mode 100644 index 0000000..9d26e24 --- /dev/null +++ b/MANAGE/setup_mongo.py @@ -0,0 +1,46 @@ +from uuid import uuid4 +from pymongo import IndexModel, MongoClient, GEOSPHERE, TEXT, errors +import os +import json + +from common import make_hash + +dirname = os.path.dirname(__file__) + +def setup_tweets(db): + text_index = IndexModel([('text', TEXT)]) + geo_index = IndexModel([('geo', GEOSPHERE)]) + db.tweets.create_indexes([text_index, geo_index]) + +def setup_roles(db): + with open(os.path.join(dirname, './seeds/roles.json')) as f: + roles = json.load(f) + insert_ignore_dupes(db.roles, roles) + +def setup_users(db, salt): + if os.environ.get('ADD_DEBUG_RECORDS') == 'True': + print('Inserting test users into db') + with open(os.path.join(dirname, './seeds/users.json')) as f: + users = json.load(f) + hasher = make_hash(salt['algorithm'], salt['iterations'], salt['key_length']) + for user in users: + user['password'] = hasher(user['password'], uuid4().hex) + insert_ignore_dupes(db.users, users) + +def insert_ignore_dupes(collection, records): + try: + collection.insert_many(records, ordered = False) + except errors.BulkWriteError as bwe: + pass + +client = MongoClient(host = os.environ.get('MONGO_HOST'), port = int(os.environ.get('MONGO_PORT'))) + +settings = {} +constantsFile = os.path.join(dirname, '../SHARED/json/constants.json') +with open(constantsFile) as f: + settings = json.load(f) + +db = client[settings['database_name']] +setup_tweets(db) +setup_roles(db) +setup_users(db, settings['salt']) diff --git a/SHARED/__init__.py b/SHARED/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SHARED/constants.json b/SHARED/json/constants.json similarity index 98% rename from SHARED/constants.json rename to SHARED/json/constants.json index 9ac7d86..582e2df 100644 --- a/SHARED/constants.json +++ b/SHARED/json/constants.json @@ -11,5 +11,5 @@ "raw_data_database_name": "raw_tweets", "max_tweets_per_get": 100, "max_historical_tweets": 5000 -} + } } diff --git a/SHARED/modules/__init__.py b/SHARED/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SHARED/modules/common.py b/SHARED/modules/common.py new file mode 100644 index 0000000..3360424 --- /dev/null +++ b/SHARED/modules/common.py @@ -0,0 +1,19 @@ +import binascii +import hashlib + +#TODO: how does CodeForPDX handle shared python modules? +def make_hash(algorithm, iterations, key_length, password = None, salt = None): + def hash_it_out(password, salt): + dk = hashlib.pbkdf2_hmac( + algorithm, + bytes(password, 'utf-8'), + bytes(salt, 'utf-8'), + iterations, + dklen=int(key_length) + ) + return binascii.hexlify(dk) + if password is None or salt is None: + return hash_it_out + else: + return hash_it_out(password, salt) + \ No newline at end of file diff --git a/TWITTER/README.md b/TWITTER/README.md index ccde03c..d604971 100644 --- a/TWITTER/README.md +++ b/TWITTER/README.md @@ -22,7 +22,7 @@ writer. They communicate through a shared in-memory queue. File Layout ----------- app.py - The main Flask app, which: - Reads the ../SHARED/constants.json file for parameters + Reads the ../SHARED/json/constants.json file for parameters Creates a queue.Queue() for communication Starts the mongo manager which triggers every 12 seconds Starts the twitter manager which triggers every diff --git a/TWITTER/app.py b/TWITTER/app.py index fd99444..46c9140 100644 --- a/TWITTER/app.py +++ b/TWITTER/app.py @@ -24,7 +24,7 @@ def read_settings(): """ Read in Twitter constants """ try: - with open('../SHARED/constants.json') as f: + with open('../SHARED/json/constants.json') as f: constants = json.load(f) settings = constants['twitter'] except Exception: diff --git a/TWITTER/mongo.py b/TWITTER/mongo.py index 38f82ee..e830d9d 100644 --- a/TWITTER/mongo.py +++ b/TWITTER/mongo.py @@ -15,7 +15,7 @@ def get_host_and_port(): Returns host and port of mongodb that we should connect to. Within the container, docker-compose has send the environment variables. On the host, we should not. This is because to - the host, the mongod is at 'localhost' and inside other + the host, the mongodb is at 'localhost' and inside other containers, it is at 'mongo' """ mongo_host = os.environ.setdefault('MONGO_HOST', 'localhost') diff --git a/WEB/app.py b/WEB/app.py index f769019..142d317 100644 --- a/WEB/app.py +++ b/WEB/app.py @@ -38,7 +38,7 @@ def create_app(testconfig=None): # Load the values from the constants file. This file contains the # parameters that are used for the hashing algorithim that is # applied to the salted passwords. - with open('../SHARED/constants.json') as f: + with open('../SHARED/json/constants.json') as f: constants = json.load(f) # Read the private key from `./keys/token`. This file is gitignored so diff --git a/docker-compose.yml b/docker-compose.yml index 7523e63..1212300 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,28 +53,6 @@ services: # - ./SHARED:/usr/src/app/SHARED #End of debugging config - manage: #defines a node enclosure for interacting with the mongo database - container_name: sms-mongodb-manager - build: - context: ./ - dockerfile: ./MANAGE/Dockerfile - depends_on: - - mongo - environment: - MONGO_HOST: mongo - MONGO_PORT: 27017 - - #Mounts the migration folder of the image to the source, to allow for dynamic updates of the migration code. - #Should only be used for debugging. - # volumes: - # - ./MANAGE/migrations:/usr/src/app/migrations - #End of debugging config - - #Causes this service to remain idle, allowing the user to use it as a proxy for pushing database updates - command: tail -F anything - - - mongo: #defines the mongoDB enclosure image: mongo:4.0.5 container_name: sms-mongodb @@ -88,5 +66,17 @@ services: ports: - 27017:27017 + manage: #defines a node enclosure for interacting with the mongo database + container_name: sms-mongodb-manager + build: + context: ./ + dockerfile: ./MANAGE/Dockerfile + depends_on: + - mongo + environment: + MONGO_HOST: mongo + MONGO_PORT: 27017 + ADD_DEBUG_RECORDS: "True" # True for debugging, inserts test users + volumes: symptomsurvey_db_volume: