Merge pull request #333 from open-craft/mtyaka/FAL-2198-django3.2

[FAL-2198] Django 3.2 support

Author: mtyaka

.gitignore

@@ -1,11 +1,12 @@
 *~
 *.pyc
-/.coverage
+/.coverage*
 /xblock_problem_builder.egg-info
-/workbench.*
+/workbench*
 /dist
 /templates
 /var
+/src
 *.iml
 .idea/*
 dump.rdb

Makefile

@@ -8,7 +8,7 @@ help: ## display this help message
 	@echo "Please use \`make <target>' where <target> is one of"
 	@perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
 
-upgrade:
+upgrade: ## Upgrade test requirements
 	pip-compile --upgrade --output-file test_requirements.txt test_requirements.in
 
 extract_translations: ## extract strings to be translated, outputting .po files
@@ -24,7 +24,7 @@ compile_translations: ## compile translation files, outputting .mo files for eac
 	cd $(WORKING_DIR) && i18n_tool generate
 	python manage.py compilejsi18n --namespace ProblemBuilderXBlockI18N --output $(JS_TARGET)
 
-detect_changed_source_translations:
+detect_changed_source_translations: ## Detect changes in source code that affect translations
 	cd $(WORKING_DIR) && i18n_tool changed
 
 dummy_translations: ## generate dummy translation (.po) files

README.md

@@ -169,6 +169,54 @@ make test.unit
 make test.integration
 ```
 
+Debugging CI Failures
+---------------------
+
+Sometimes it can be hard to figure out why some tests fail in the CI.
+When Circle CI browser based tests fail for unknown reasons, it can be helpful to run them with VNC enabled
+so that you can observe the browser (or even interact with it) while the tests are running.
+
+To enable VNC on Circle CI, first re-run the failing test with SSH enabled: in the Circle CI UI,
+click the "Rerun" dropdown and select "Rerun Job with SSH". The job will be re-run with SSH enabled.
+You can find the IP/port combination that lets you log into the VM with your github SSH key under the "Enable SSH"
+step in the pipeline UI.
+
+SSH into the VM, forwarding the VNC port:
+
+```bash
+ssh -p <port> <ip-address> -L 5900:localhost:5900
+```
+
+Install the required packages:
+
+```bash
+sudo apt-get install -yq xvfb x11vnc fluxbox
+```
+
+Start up xvfb and the VNC server:
+
+```bash
+rm -f /tmp/.X$(echo ${DISPLAY:-:0} | cut -b2-)-lock
+Xvfb ${DISPLAY:-:0} -ac -listen tcp -screen 0 1440x900x24 &
+/usr/bin/fluxbox -display ${DISPLAY:-:0} -screen 0 &
+x11vnc -display ${DISPLAY:-:0} -forever -noxdamage -rfbport 5900 -quiet -passwd pass &
+```
+
+You should now be able to connect to the server via VNC. On macOS, you can use the built-in VNC viewer
+that you can launch by opening Finder and choosing the "Go -> Connect to Server.." from the menu.
+Type in `localhost:5900` and enter `pass` when asked for the password.
+
+You are all set up to run integration tests with screen sharing enabled.
+For some reason Firefox does not want to start in foreground mode when run as non-root,
+so you'll have to run the tests as root.
+
+```bash
+unset MOZ_HEADLESS
+cd /home/circleci/project
+source venv/bin/activate
+make test
+```
+
 Working with Translations
 -------------------------
 

circle.yml

@@ -1,9 +1,11 @@
 version: 2.1
+orbs:
+  browser-tools: circleci/[email protected]
 jobs:
   build:
     docker:
       - image: <<parameters.docker_image>>
-      - image: circleci/mysql:5.6
+      - image: circleci/mysql:5.7
         command: mysqld --character-set-server=latin1 --collation-server=latin1_swedish_ci
         environment:
           MYSQL_ROOT_PASSWORD: rootpw
@@ -12,29 +14,17 @@ jobs:
             type: string
         docker_image:
             type: string
+        parallel:
+            default: 1
+            type: integer
+    parallelism: <<parameters.parallel>>
     environment:
       MOZ_HEADLESS: 1
       WORKBENCH_DATABASES: '{"default": {"ENGINE": "django.db.backends.mysql", "NAME": "db", "USER": "root", "PASSWORD": "rootpw", "HOST": "127.0.0.1", "OPTIONS": {"charset": "utf8mb4"}}}'
     steps:
       - checkout
-      - run:
-          name: Update system
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y libgtk3.0-cil-dev libasound2 libasound2 libdbus-glib-1-2 libdbus-1-3 libgtk2.0-0 default-libmysqlclient-dev
-      - run:
-          name: Install geckodriver
-          command: |
-            wget https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz
-            tar xzf geckodriver-v0.26.0-linux64.tar.gz
-            sudo mv geckodriver /usr/local/bin/geckodriver
-            which geckodriver
-      - run:
-          name: Install new firefox
-          command: |
-            wget https://archive.mozilla.org/pub/firefox/releases/70.0.1/linux-x86_64/en-US/firefox-70.0.1.tar.bz2
-            tar jxf firefox-70.0.1.tar.bz2
-            export PATH="$(pwd)/firefox/:$PATH"
+      - browser-tools/install-firefox
+      - browser-tools/install-geckodriver
       - run:
           name: Sync submodules
           command: git submodule sync
@@ -44,43 +34,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            export PATH="$(pwd)/firefox/:$PATH"
-            mkdir -p /tmp/coverage/$CIRCLE_SHA1
             virtualenv venv
             source venv/bin/activate
+            mkdir var
+            pip -q install -r requirements.txt
+            pip -q install -r requirements-dev.txt
             pip -q install -r test_requirements.txt
             pip install -e .
             <<parameters.test_command>>
-            if [ -e .coverage.* ]; then
-                cp .coverage.* /tmp/coverage/$CIRCLE_SHA1/.
-            fi
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-              - coverage
-
-  coverage:
-    docker:
-      - image: <<parameters.docker_image>>
-    parameters:
-        docker_image:
-            type: string
-    steps:
-      - checkout
-      - run:
-          name: Update system and install deps
-          command: |
-            sudo apt-get update
-            virtualenv venv
-            source venv/bin/activate
-            pip install 'coverage==5.0.3'
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          command: |
-              source venv/bin/activate
-              coverage combine /tmp/workspace/coverage/$CIRCLE_SHA1/
-              coverage report
 
   deploy:
     docker:
@@ -131,42 +92,38 @@ workflows:
   build_and_deploy:
     jobs:
       - build:
-          name: py35-quality
+          name: py38-quality
           test_command: make quality
-          docker_image: circleci/python:3.5-buster-browsers
+          docker_image: cimg/python:3.8-browsers
           filters:
             tags:
               only: /.*/
       - build:
-          name: py35-unit
+          name: py38-unit
           test_command: make test.unit
-          docker_image: circleci/python:3.5-buster-browsers
+          docker_image: cimg/python:3.8-browsers
           filters:
             tags:
               only: /.*/
       - build:
-          name: py35-integration
-          test_command: make test.integration
-          docker_image: circleci/python:3.5-buster-browsers
-          filters:
-            tags:
-              only: /.*/
-      - coverage:
-          name: py35-coverage
-          docker_image: circleci/python:3.5-buster
+          name: py38-integration
+          parallel: 4
+          test_command: |
+            set -e
+            TEST_FILES=$(circleci tests glob "problem_builder/tests/integration/test_*.py" | circleci tests split --split-by=timings)
+            pytest --verbose  $TEST_FILES
+          docker_image: cimg/python:3.8-browsers
           filters:
             tags:
               only: /.*/
-          requires:
-              - py35-quality
-              - py35-unit
-              - py35-integration
       - deploy:
-          name: py35-deploy-bdist_wheel
-          docker_image: circleci/python:3.5-buster
+          name: py38-deploy-bdist_wheel
+          docker_image: cimg/python:3.8
           dist_type: bdist_wheel
           requires:
-            - py35-coverage
+            - py38-quality
+            - py38-unit
+            - py38-integration
           filters:
             tags:
               only: /v[0-9]+(\.[0-9]+)*/

problem_builder/answer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Copyright (c) 2014-2015 Harvard, edX & OpenCraft
 #
@@ -23,12 +22,11 @@
 import uuid
 
 import pkg_resources
-import six
-from lazy import lazy
 from django import utils
+from lazy import lazy
+from web_fragments.fragment import Fragment
 from xblock.core import XBlock
 from xblock.fields import Integer, Scope, String
-from xblock.fragment import Fragment
 from xblock.validation import ValidationMessage
 from xblockutils.resources import ResourceLoader
 from xblockutils.studio_editable import (StudioEditableXBlockMixin,
@@ -60,7 +58,7 @@ class AnswerMixin(XBlockWithPreviewMixin, XBlockWithTranslationServiceMixin, Stu
     Mixin to give an XBlock the ability to read/write data to the Answers DB table.
     """
     def build_user_state_data(self, context=None):
-        result = super(AnswerMixin, self).build_user_state_data()
+        result = super().build_user_state_data()
         result['student_input'] = self.student_input
         return result
 
@@ -131,13 +129,13 @@ def validate_field_data(self, validation, data):
         """
         Validate this block's field data.
         """
-        super(AnswerMixin, self).validate_field_data(validation, data)
+        super().validate_field_data(validation, data)
 
         def add_error(msg):
             validation.add(ValidationMessage(ValidationMessage.ERROR, msg))
 
         if not data.name:
-            add_error(u"A Question ID is required.")
+            add_error("A Question ID is required.")
 
 
 @XBlock.needs("i18n")
@@ -150,7 +148,7 @@ class AnswerBlock(SubmittingXBlockMixin, AnswerMixin, QuestionMixin, StudioEdita
     to make them searchable and referenceable across xblocks.
     """
     CATEGORY = 'pb-answer'
-    STUDIO_LABEL = _(u"Long Answer")
+    STUDIO_LABEL = _("Long Answer")
     answerable = True
 
     name = String(
@@ -188,11 +186,10 @@ def resource_string(path):
         return data.decode("utf8")
 
     def get_translation_content(self):
+        lang = utils.translation.to_locale(utils.translation.get_language())
         try:
-            return self.resource_string('public/js/translations/{lang}/textjs.js'.format(
-                lang=utils.translation.to_locale(utils.translation.get_language()),
-            ))
-        except IOError:
+            return self.resource_string(f'public/js/translations/{lang}/textjs.js')
+        except OSError:
             return self.resource_string('public/js/translations/en/textjs.js')
 
     def mentoring_view(self, context=None):
@@ -245,7 +242,8 @@ def submit(self, submission):
             item_key['item_id'] = self.name
             sub_api.create_submission(item_key, self.student_input)
 
-        log.info(u'Answer submitted for`{}`: "{}"'.format(self.name, self.student_input))
+        log_message = f'Answer submitted for`{self.name}`: "{self.student_input}"'
+        log.info(log_message)
         return self.get_results()
 
     @property
@@ -264,7 +262,7 @@ def save(self):
         """
         Replicate data changes on the related Django model used for sharing of data accross XBlocks
         """
-        super(AnswerBlock, self).save()
+        super().save()
 
         student_id = self._get_student_id()
         if not student_id:
@@ -294,7 +292,7 @@ def student_view_data(self, context=None):
         """
         return {
             'id': self.name,
-            'block_id': six.text_type(self.scope_ids.usage_id),
+            'block_id': str(self.scope_ids.usage_id),
             'display_name': self.display_name,
             'type': self.CATEGORY,
             'weight': self.weight,
@@ -310,7 +308,7 @@ class AnswerRecapBlock(AnswerMixin, StudioEditableXBlockMixin, XBlock):
     """
 
     CATEGORY = 'pb-answer-recap'
-    STUDIO_LABEL = _(u"Long Answer Recap")
+    STUDIO_LABEL = _("Long Answer Recap")
 
     name = String(
         display_name=_("Question ID"),
@@ -343,9 +341,9 @@ def mentoring_view(self, context=None):
             location = self.location.replace(branch=None, version=None)  # Standardize the key in case it isn't already
             target_key = {
                 'student_id': student_submissions_key,
-                'course_id': six.text_type(location.course_key),
+                'course_id': str(location.course_key),
                 'item_id': self.name,
-                'item_type': u'pb-answer',
+                'item_type': 'pb-answer',
             }
             submissions = sub_api.get_submissions(target_key, limit=1)
             try:
@@ -381,6 +379,6 @@ def student_view_data(self, context=None):
             'name': self.name,  # For backwards compatibility; same as 'id'
             'display_name': self.display_name,
             'description': self.description,
-            'block_id': six.text_type(self.scope_ids.usage_id),
+            'block_id': str(self.scope_ids.usage_id),
             'type': self.CATEGORY
         }