Migrate to Cloud NDB to prepare for Python3 upgrade

This implementation is based on:
https://cloud.google.com/appengine/docs/standard/python3/migrating-to-cloud-ndb

- [x] Replaced Python 2 only NDB library with Cloud NDB `from google.cloud import ndb`
- [x] Introduces `appengine_config.py` to include third party libraries, which are not supported by GAE natively.
- [x] Includes `client_secret.json` in the App deployment to allow access to Datastore from app engine
- [x] Got rid of `travis.py` and added instructions to run tests via simple commands
- [x] Configured requests to use URLFetch See [this](https://cloud.google.com/appengine/docs/standard/python/issue-requests#requests)
- [x] Added basic logging for debugging

We have two requirements file at the moment:
- `local_requirements.txt`: This is required for local development only, See [this](https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27#local_development)
- `requirements.txt`: This is installed in the `lib` folder for local as well as deployed environment.

Changes from Google App engine console:
- [x] Added the role `Cloud Datastore Owner` to the service account to be able to access Datastore from app engine App.

Author: aktech

.gitignore

@@ -30,3 +30,6 @@ my/
 
 # Backup files
 *~
+
+# IntelliJ/Jetbrains
+.idea

.travis.yml

@@ -7,28 +7,34 @@ virtualenv:
 
 before_install:
   - npm install -g casperjs
-install: pip install -r requirements.txt
+  - pip install nose
+install:
+  - pip install -r requirements.txt -t lib/
+  - pip install -r local_requirements.txt
 
 before_script:
+  - openssl aes-256-cbc -K $encrypted_2fd045226a67_key -iv $encrypted_2fd045226a67_iv
+      -in client-secret.json.enc -out client-secret.json -d
   - cd ..
   - wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.90.zip -nv
   - unzip -q google_appengine_1.9.90.zip
   - export SDK_LOCATION="$(pwd)/google_appengine"
   - cd $TRAVIS_BUILD_DIR
   - python $SDK_LOCATION/dev_appserver.py --skip_sdk_update_check 1 . &
   - sleep 10
+
 script:
-  - python travis.py
+  - PYTHONPATH='.' nosetests app/test -vv
+  - casperjs test app/test
 
 before_deploy:
-  - openssl aes-256-cbc -K $encrypted_2fd045226a67_key -iv $encrypted_2fd045226a67_iv
-    -in client-secret.json.enc -out ../client-secret.json -d
   - version=$(if [ ! -z "$TRAVIS_TAG" ]; then echo $(cut -d'-' -f2 <<<"$TRAVIS_TAG"); else echo "$TRAVIS_BRANCH"; fi)
   - echo "Version = $version"
 deploy:
   provider: gae
-  keyfile: "../client-secret.json"
+  keyfile: "client-secret.json"
   project: sympy-gamma-hrd
+  skip_cleanup: true
   no_promote: true
   version: "$version"
   on:

README.rst

@@ -46,6 +46,22 @@ We use submodules to include external libraries in sympy_gamma::
 This is sufficient to clone appropriate repositories in correct versions
 into sympy_gamma (see git documentation on submodules for information).
 
+Install Dependencies
+--------------------
+
+The project depends on some third-party libraries that are not on the list
+of built-in libraries (in app.yaml) bundled with the runtime, to install them
+run the following command.::
+
+    pip install -r requirements.txt -t lib/
+
+Some libraries although available on app engine runtime, but needs to be
+installed locally for development.
+
+Ref: https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27#local_development ::
+
+    pip install -r local_requirements.txt
+
 Development server
 ------------------
 
@@ -173,6 +189,24 @@ NumPy version), change ``app.yaml.template`` and generate again. The
 Travis-CI script uses this to generate and deploy testing/production
 versions automatically.
 
+
+Running Tests
+-------------
+
+To be able to run tests, make sure you have testing libraries installed::
+
+    npm install -g casperjs
+    pip install nose
+
+To run unit tests::
+
+    PYTHONPATH='.' nosetests app/test -vv
+
+To run PhantomJS Tests::
+
+    casperjs test app/test
+
+
 Pulling changes
 ---------------
 

app.yaml

@@ -29,3 +29,8 @@ libraries:
   version: "1.6.1"
 - name: ssl
   version: "latest"
+- name: grpcio
+  version: "1.0.0"
+
+env_variables:
+  GOOGLE_APPLICATION_CREDENTIALS: "client-secret.json"

app/models.py

@@ -1,4 +1,11 @@
-from google.appengine.ext import ndb
+import six
+# https://github.com/googleapis/python-ndb/issues/249#issuecomment-560957294
+six.moves.reload_module(six)
+
+from google.cloud import ndb
+
+ndb_client = ndb.Client()
+
 
 class Query(ndb.Model):
     text = ndb.StringProperty()

app/views.py

@@ -6,7 +6,7 @@
 from google.appengine.api import users
 from google.appengine.runtime import DeadlineExceededError
 
-from logic.logic import SymPyGamma, mathjax_latex
+from logic.logic import SymPyGamma
 
 import settings
 import models
@@ -19,6 +19,12 @@
 import datetime
 import traceback
 
+import logging
+
+
+ndb_client = models.ndb_client
+
+
 LIVE_URL = '<a href="https://live.sympy.org">SymPy Live</a>'
 LIVE_PROMOTION_MESSAGES = [
     'Need more control? Try ' + LIVE_URL + '.',
@@ -180,8 +186,9 @@ def index(request, user):
     form = SearchForm()
 
     if user:
-        history = models.Query.query(models.Query.user_id==user.user_id())
-        history = history.order(-models.Query.date).fetch(10)
+        with ndb_client.context():
+            history = models.Query.query(models.Query.user_id == user.user_id())
+            history = history.order(-models.Query.date).fetch(10)
     else:
         history = None
 
@@ -193,9 +200,23 @@ def index(request, user):
         "examples": EXAMPLES
         })
 
+
+def user_exists_and_input_not_present(user, input):
+    with ndb_client.context():
+        return (user and not models.Query.query(
+            models.Query.text == input,
+            models.Query.user_id == user.user_id()).get())
+
+
+def input_exists(input):
+    with ndb_client.context():
+        return models.Query.query(models.Query.text == input).get()
+
+
 @app_meta
 @authenticate
 def input(request, user):
+    logging.info('Got the input from user')
     if request.method == "GET":
         form = SearchForm(request.GET)
         if form.is_valid():
@@ -214,15 +235,18 @@ def input(request, user):
                     "output": "Can't handle the input."
                 }]
 
-            if (user and not models.Query.query(
-                    models.Query.text==input,
-                    models.Query.user_id==user.user_id()).get()):
-                query = models.Query(text=input, user_id=user.user_id())
-                query.put()
-            elif not models.Query.query(models.Query.text==input).get():
-                query = models.Query(text=input, user_id=None)
-                query.put()
-
+            if user_exists_and_input_not_present(user, input):
+                logging.info('User exists and input not present')
+                with ndb_client.context():
+                    query = models.Query(text=input, user_id=user.user_id())
+                    logging.info('query: %s' % query)
+                    query.put()
+            elif not input_exists(input):
+                logging.info('Input does not exists')
+                with ndb_client.context():
+                    query = models.Query(text=input, user_id=None)
+                    logging.info('query: %s' % query)
+                    query.put()
 
             # For some reason the |random tag always returns the same result
             return ("result.html", {
@@ -362,17 +386,25 @@ def get_card_full(request, card_name):
     return response
 
 
+def find_text_query(query):
+    with ndb_client.context():
+        return models.Query.query(models.Query.text == query.text)
+
+
 def remove_query(request, qid):
     user = users.get_current_user()
 
     if user:
-        query = models.ndb.Key(urlsafe=qid).get()
+        with ndb_client.context():
+            query = models.ndb.Key(urlsafe=qid).get()
 
-        if not models.Query.query(models.Query.text==query.text):
-            query.user_id = None
-            query.put()
+        if not find_text_query(query):
+            with ndb_client.context():
+                query.user_id = None
+                query.put()
         else:
-            query.key.delete()
+            with ndb_client.context():
+                query.key.delete()
 
         response = {
             'result': 'success',

appengine_config.py

@@ -0,0 +1,11 @@
+import os
+import pkg_resources
+from google.appengine.ext import vendor
+
+# Set path to your libraries folder.
+path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')
+# path = 'lib'
+# Add libraries installed in the path folder.
+vendor.add(path)
+# Add libraries to pkg_resources working set to find the distribution.
+pkg_resources.working_set.add_entry(path)

local_requirements.txt

@@ -0,0 +1,3 @@
+numpy==1.6.1
+protobuf
+enum34

main.py

@@ -9,5 +9,9 @@
 
 from django.core.wsgi import get_wsgi_application
 
-application = get_wsgi_application()
+# https://cloud.google.com/appengine/docs/standard/python/issue-requests#requests
+import requests_toolbelt.adapters.appengine
+# Use the App Engine Requests adapter. This makes sure that Requests uses URLFetch.
+requests_toolbelt.adapters.appengine.monkeypatch()
 
+application = get_wsgi_application()

requirements.txt

@@ -1,2 +1,3 @@
-django==1.11
-numpy==1.6.1
+googleapis_common_protos
+google-cloud-ndb
+requests-toolbelt