Core: Add memory to the runEnd event to allow late event listeners

While this is not in any way a documented API, it is quite common for
QUnit runners that are based on Selenium/Webdriver to use HTML scraping
to obtain a simple summary of whether the run passed.

Intruce memory for `runEnd` event so that anything JS-based has a
stable and documented way to obtain the result in a machine-readable
format, without needing to resort to HTML scraping.

One common reason sometimes avoid the event emitter is that, depending
on various circumstances, a custom test runner might end up injecting
code slightly too late, at which point JS code to listen for our event
might not fire, whereas the DOM is still available. This is rare,
but can happen:

* if the file is served as-is.

  browserstack-runner and QTap solves this by proxying the HTML file
  and inject an inline script to reliable listen early.

  Karma solves this by proxing the test framework instead (qunit.js)
  and adapting it to include relevant event listeners upfront.

* and, if the browser is driven without early JS control.
  When WebDriver v1 is used, something like webdriver "execute" is
  not guruanteed to run before DOM-ready or window.onload. If the
  Node.js process is slow or far away from the browser (e.g. cloud),
  and if the test suite is relatively small/fast, then the injected
  code might arrive after the tests have already finished.

  grunt-contrib-qunit avoids this by using Puppeteer and
  its `Page.evaluateOnNewDocument()` method, to reliably run a script
  before any others.

* and, if the test is relatively small/fast.

Example at mauriciolauffer/wdio-qunit-service#13

Solve this once and for all by adding memory to the `runEnd` event.
This allows late event listers to handle the event retroactively.

Cherry-picked from 8f25f26 (3.0.0-dev).

Author: Krinkle

src/core/config.js

@@ -131,6 +131,8 @@ const config = {
   // started: 0,
 
   // Internal state
+  _event_listeners: Object.create(null),
+  _event_memory: {},
   _deprecated_timeout_shown: false,
   _deprecated_countEachStep_shown: false,
   blocking: true,

src/events.js

@@ -1,6 +1,6 @@
 import { inArray } from './core/utilities';
+import config from './core/config';
 
-const LISTENERS = Object.create(null);
 const SUPPORTED_EVENTS = [
   'error',
   'runStart',
@@ -11,6 +11,9 @@ const SUPPORTED_EVENTS = [
   'suiteEnd',
   'runEnd'
 ];
+const MEMORY_EVENTS = [
+  'runEnd'
+];
 
 /**
  * Emits an event with the specified data to all currently registered listeners.
@@ -30,12 +33,16 @@ export function emit (eventName, data) {
   }
 
   // Clone the callbacks in case one of them registers a new callback
-  const originalCallbacks = LISTENERS[eventName];
+  const originalCallbacks = config._event_listeners[eventName];
   const callbacks = originalCallbacks ? [...originalCallbacks] : [];
 
   for (let i = 0; i < callbacks.length; i++) {
     callbacks[i](data);
   }
+
+  if (inArray(MEMORY_EVENTS, eventName)) {
+    config._event_memory[eventName] = data;
+  }
 }
 
 /**
@@ -57,12 +64,14 @@ export function on (eventName, callback) {
     throw new TypeError('callback must be a function when registering a listener');
   }
 
-  if (!LISTENERS[eventName]) {
-    LISTENERS[eventName] = [];
-  }
+  const listeners = config._event_listeners[eventName] || (config._event_listeners[eventName] = []);
 
   // Don't register the same callback more than once
-  if (!inArray(callback, LISTENERS[eventName])) {
-    LISTENERS[eventName].push(callback);
+  if (!inArray(callback, listeners)) {
+    listeners.push(callback);
+
+    if (config._event_memory[eventName] !== undefined) {
+      callback(config._event_memory[eventName]);
+    }
   }
 }

test/cli/fixtures/event-runEnd-memory.js

@@ -0,0 +1,23 @@
+QUnit.on('runEnd', function (run) {
+  console.log(`# early runEnd total=${run.testCounts.total} passed=${run.testCounts.passed} failed=${run.testCounts.failed}`);
+  setTimeout(function () {
+    QUnit.on('runEnd', function (run) {
+      console.log(`# late runEnd total=${run.testCounts.total} passed=${run.testCounts.passed} failed=${run.testCounts.failed}`);
+    });
+  });
+});
+
+QUnit.module('First', function () {
+  QUnit.test('A', function (assert) {
+    assert.true(true);
+  });
+  QUnit.test('B', function (assert) {
+    assert.true(false);
+  });
+});
+
+QUnit.module('Second', function () {
+  QUnit.test('C', function (assert) {
+    assert.true(true);
+  });
+});

test/cli/fixtures/event-runEnd-memory.tap.txt

@@ -0,0 +1,23 @@
+# command: ["qunit", "event-runEnd-memory.js"]
+
+TAP version 13
+ok 1 First > A
+not ok 2 First > B
+  ---
+  message: failed
+  severity: failed
+  actual  : false
+  expected: true
+  stack: |
+        at /qunit/test/cli/fixtures/event-runEnd-memory.js:15:16
+  ...
+ok 3 Second > C
+1..3
+# pass 2
+# skip 0
+# todo 0
+# fail 1
+# early runEnd total=3 passed=2 failed=1
+# late runEnd total=3 passed=2 failed=1
+
+# exit code: 1