Python mongo migration

MANAGE/Dockerfile

@@ -1,9 +1,17 @@
-FROM node:7.10.1-slim
+FROM python:3.6-slim
+
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.name="symptomsurvey_backed/DB"
+LABEL org.label-schema.description="Scripts for setting up the database used by the other sections of project"
+LABEL org.label-schema.url="http://www.codeforpdx.org/projects/2"
+LABEL org.label-schema.vcs-url="https://github.com/codeforpdx/symptomsurvey_backend"
 
 WORKDIR /usr/src/app
 
-#do this BEFORE copying the rest, that way only changes to package.json will cause npm to execute
-COPY ./MANAGE/package.json ./package.json
-RUN npm install
+RUN pip install --upgrade pip
+RUN pip3 install pymongo
 
-COPY ./MANAGE ./SHARED ./
+COPY ./MANAGE /MANAGE
+COPY ./SHARED/modules/common.py /MANAGE/common.py
+COPY ./SHARED/json /SHARED/json
+CMD ["python", "/MANAGE/setup_mongo.py"]

MANAGE/setup_mongo.py

@@ -0,0 +1,46 @@
+from uuid import uuid4
+from pymongo import IndexModel, MongoClient, GEOSPHERE, TEXT, errors
+import os
+import json
+
+from common import make_hash
+
+dirname = os.path.dirname(__file__)
+
+def setup_tweets(db):
+    text_index = IndexModel([('text', TEXT)])
+    geo_index = IndexModel([('geo', GEOSPHERE)])
+    db.tweets.create_indexes([text_index, geo_index])
+
+def setup_roles(db):
+    with open(os.path.join(dirname, './seeds/roles.json')) as f:
+        roles = json.load(f)
+        insert_ignore_dupes(db.roles, roles)
+
+def setup_users(db, salt):
+    if os.environ.get('ADD_DEBUG_RECORDS') == 'True':
+        print('Inserting test users into db')
+        with open(os.path.join(dirname, './seeds/users.json')) as f:
+            users = json.load(f)
+            hasher = make_hash(salt['algorithm'], salt['iterations'], salt['key_length'])
+            for user in users:
+                user['password'] = hasher(user['password'], uuid4().hex)
+            insert_ignore_dupes(db.users, users)
+
+def insert_ignore_dupes(collection, records):
+    try:
+        collection.insert_many(records, ordered = False)
+    except errors.BulkWriteError as bwe:
+        pass
+
+client = MongoClient(host = os.environ.get('MONGO_HOST'), port = int(os.environ.get('MONGO_PORT')))
+
+settings = {}
+constantsFile = os.path.join(dirname, '../SHARED/json/constants.json')
+with open(constantsFile) as f:
+    settings = json.load(f)
+
+db = client[settings['database_name']]
+setup_tweets(db)
+setup_roles(db)
+setup_users(db, settings['salt'])

SHARED/constants.json → SHARED/json/constants.json

@@ -11,5 +11,5 @@
     "raw_data_database_name": "raw_tweets",
     "max_tweets_per_get": 100,
     "max_historical_tweets": 5000
-}
+  }
 }

SHARED/modules/common.py

@@ -0,0 +1,19 @@
+import binascii
+import hashlib
+
+#TODO: how does CodeForPDX handle shared python modules?
+def make_hash(algorithm, iterations, key_length, password = None, salt = None):
+    def hash_it_out(password, salt):
+        dk = hashlib.pbkdf2_hmac(
+            algorithm,
+            bytes(password, 'utf-8'),
+            bytes(salt, 'utf-8'),
+            iterations,
+            dklen=int(key_length)
+        )
+        return binascii.hexlify(dk)
+    if password is None or salt is None:
+        return hash_it_out
+    else:
+        return hash_it_out(password, salt)
+

TWITTER/README.md

@@ -22,7 +22,7 @@ writer.  They communicate through a shared in-memory queue.
 File Layout
 -----------
 app.py - The main Flask app, which:
-    Reads the ../SHARED/constants.json file for parameters
+    Reads the ../SHARED/json/constants.json file for parameters
     Creates a queue.Queue() for communication
     Starts the mongo manager which triggers every 12 seconds
     Starts the twitter manager which triggers every

TWITTER/app.py

@@ -24,7 +24,7 @@
 def read_settings():
     """ Read in Twitter constants """
     try:
-        with open('../SHARED/constants.json') as f:
+        with open('../SHARED/json/constants.json') as f:
             constants = json.load(f)
         settings = constants['twitter']
     except Exception:

TWITTER/mongo.py

@@ -15,7 +15,7 @@ def get_host_and_port():
     Returns host and port of mongodb that we should connect to.
     Within the container, docker-compose has send the environment
     variables.  On the host, we should not.  This is because to
-    the host, the mongod is at 'localhost' and inside other
+    the host, the mongodb is at 'localhost' and inside other
     containers, it is at 'mongo'
     """
     mongo_host = os.environ.setdefault('MONGO_HOST', 'localhost')

WEB/app.py

@@ -38,7 +38,7 @@ def create_app(testconfig=None):
     # Load the values from the constants file.  This file contains the
     # parameters that are used for the hashing algorithim that is
     # applied to the salted passwords.
-    with open('../SHARED/constants.json') as f:
+    with open('../SHARED/json/constants.json') as f:
       constants = json.load(f)
 
     # Read the private key from `./keys/token`. This file is gitignored so

docker-compose.yml

@@ -53,28 +53,6 @@ services:
     #  - ./SHARED:/usr/src/app/SHARED
     #End of debugging config
 
-  manage: #defines a node enclosure for interacting with the mongo database
-    container_name: sms-mongodb-manager
-    build:
-      context: ./
-      dockerfile: ./MANAGE/Dockerfile
-    depends_on:
-      - mongo
-    environment:
-      MONGO_HOST: mongo
-      MONGO_PORT: 27017
-
-    #Mounts the migration folder of the image to the source, to allow for dynamic updates of the migration code.
-    #Should only be used for debugging.
-    # volumes:
-    #   - ./MANAGE/migrations:/usr/src/app/migrations
-    #End of debugging config
-
-    #Causes this service to remain idle, allowing the user to use it as a proxy for pushing database updates
-    command: tail -F anything
-
-
-
   mongo: #defines the mongoDB enclosure
     image: mongo:4.0.5
     container_name: sms-mongodb
@@ -88,5 +66,17 @@ services:
     ports:
       - 27017:27017
 
+  manage: #defines a node enclosure for interacting with the mongo database
+    container_name: sms-mongodb-manager
+    build:
+      context: ./
+      dockerfile: ./MANAGE/Dockerfile
+    depends_on:
+      - mongo
+    environment:
+      MONGO_HOST: mongo
+      MONGO_PORT: 27017
+      ADD_DEBUG_RECORDS: "True" # True for debugging, inserts test users
+
 volumes:
   symptomsurvey_db_volume: