Merge pull request #4588 from manosnoam/cy_retry

Add CY_RETRY option for Cypress test retry configuration

Author: manosnoam

docs/testing.md

@@ -222,6 +222,72 @@ npm run cypress:open:record
 npm run cypress:run:record
 ```
 
+### Cypress Environment Variables
+
+Cypress uses several environment variables to control test behavior and configuration. These can be set in your shell or passed directly to the Cypress commands.
+
+#### CY_TEST_CONFIG
+- **Purpose**: Path to test configuration YAML file
+- **Values**: File path to YAML configuration (e.g., `./test-variables.yml`)
+- **Effect**: Loads test variables and configuration from a YAML file
+- **Usage**: `export CY_TEST_CONFIG='PATH_TO_YOUR_TEST_VARIABLES'`
+- **Required**: Yes, for e2e tests
+
+#### CY_MOCK
+- **Purpose**: Enables mocked test mode
+- **Values**: `1` (enabled) or `0`/unset (disabled)
+- **Effect**:
+  - Runs tests from `cypress/tests/mocked/` directory instead of `cypress/tests/e2e/`
+  - Changes results directory to `results/mocked/`
+  - Disables test retries by default
+  - Loads different environment files (`.env.cypress.mock`)
+- **Usage**: `CY_MOCK=1 npm run cypress:run`
+
+#### CY_RETRY
+- **Purpose**: Controls test retry behavior
+- **Values**: Number of retries (e.g., `0`, `1`, `2`)
+- **Default**: `2` retries for e2e tests, `0` for mocked tests
+- **Effect**: Sets the number of additional attempts a test will retry if it fails
+- **Usage**: `CY_RETRY=0 npm run cypress:run` (no retries)
+
+#### CY_RECORD
+- **Purpose**: Enables snapshot recording mode
+- **Values**: `1` (enabled) or `0`/unset (disabled)
+- **Effect**: Records network responses for snapshot testing
+- **Usage**: `CY_RECORD=1 npm run cypress:run`
+
+#### CY_WATCH
+- **Purpose**: Controls file watching behavior
+- **Values**: `false` to disable, any other value to enable
+- **Effect**: Enables/disables watching for file changes during test execution
+- **Usage**: `CY_WATCH=false npm run cypress:open`
+
+#### CY_WS_PORT
+- **Purpose**: Sets the WebSocket server port for mocked tests
+- **Values**: Port number (e.g., `9002`)
+- **Default**: Used in mocked test scripts
+- **Effect**: Configures the WebSocket server port for real-time communication in mocked tests
+- **Usage**: Set automatically in npm scripts like `cypress:run:mock`
+
+#### CY_COVERAGE
+- **Purpose**: Enables code coverage collection
+- **Values**: `true` (enabled) or `false`/unset (disabled)
+- **Effect**: Enables code coverage reporting during test execution
+- **Usage**: `CY_COVERAGE=true npm run cypress:run`
+
+#### CY_RESULTS_DIR
+- **Purpose**: Sets the directory for test results
+- **Values**: Directory path (e.g., `results`, `custom-results`)
+- **Default**: `results`
+- **Effect**: Changes where test results (screenshots, videos, reports) are stored
+- **Usage**: `CY_RESULTS_DIR=custom-results npm run cypress:run`
+
+#### CY_TEST_TIMEOUT_SECONDS
+- **Purpose**: Sets global test timeout
+- **Values**: Number of seconds
+- **Effect**: Configures the global timeout for all tests
+- **Usage**: `CY_TEST_TIMEOUT_SECONDS=300 npm run cypress:run`
+
 ### Page Objects
 
 Page Objects are TypeScript objects / classes used to encapsulate all the selectors and functions related to a particular page or UI element. The functions exposed by a page object use the following prefixes:
@@ -290,7 +356,7 @@ cy.wsK8s('DELETED', ProjectModel, <project resource>);
 
 Always start a new test with a `visit` to the page being tested.
 
-Use variants of `intercept` to mock network requests. 
+Use variants of `intercept` to mock network requests.
 
 When a UI action results in a network request, the test must wait to ensure the request was issued:
 ```ts

frontend/src/__tests__/cypress/cypress.config.ts

@@ -72,10 +72,32 @@ export default defineConfig({
       : `cypress/tests/e2e/**/*.cy.ts`,
     experimentalInteractiveRunEvents: true,
     setupNodeEvents(on, config) {
+      // Dynamically merge all CY_ variables from both process.env and --env sources, with --env taking precedence
+      // This allows setting CY_ variables via shell environment (CY_RETRY=1) or Cypress --env flag (--env CY_RETRY=1)
+      // Update the existing env variable with merged values
+      Object.assign(env, {
+        ...Object.fromEntries(
+          Object.keys(config.env)
+            .filter((key) => key.startsWith('CY_'))
+            .map((key) => [key, config.env[key]]),
+        ),
+        WS_PORT: (config.env.CY_WS_PORT ?? env.CY_WS_PORT ?? '9002').toString(),
+      });
+
+      // Create config with merged environment variables
+      // config.env (--env vars) takes precedence over process.env (shell env vars)
+      const configWithEnv = {
+        ...config,
+        env: {
+          ...env,
+          ...config.env,
+        },
+      };
+
       registerCypressGrep(config);
       cypressHighResolution(on, config);
       coverage(on, config);
-      setupWebsockets(on, config);
+      setupWebsockets(on, configWithEnv);
 
       on('task', {
         readJSON(filePath: string) {
@@ -148,10 +170,19 @@ export default defineConfig({
         await mergeFiles(outputFile, inputFiles);
       });
 
-      // Apply retries only for tests in the "e2e" folder
+      // Apply retries only for tests in the "e2e" folder. 2 retries by default, after a test failure.
+      // Set CY_RETRY=N env var for the number of retries (CY_RETRY=0 means no retries).
+      const retryConfig =
+        env.CY_RETRY !== undefined
+          ? { runMode: Math.max(0, parseInt(env.CY_RETRY) || 0), openMode: 0 }
+          : !env.CY_MOCK && !env.CY_RECORD
+          ? { runMode: 2, openMode: 0 }
+          : config.retries;
+
       return {
         ...config,
-        retries: !env.CY_MOCK && !env.CY_RECORD ? { runMode: 2, openMode: 0 } : config.retries,
+        retries: retryConfig,
+        env,
       };
     },
   },

frontend/src/__tests__/cypress/cypress/support/e2e.ts

@@ -105,23 +105,21 @@ Cypress.Keyboard.defaults({
   keystrokeDelay: 0,
 });
 
-// Handle ChunkLoadError - ignore these errors as they are usually due to code splitting issues during development
+// Uncaught exceptions handler
 Cypress.on('uncaught:exception', (err) => {
-  // Returning false here prevents Cypress from failing the test
+  // Handle ChunkLoadError - ignore these errors as they are usually due to code splitting issues during development
   if (err.name === 'ChunkLoadError') {
     // eslint-disable-next-line no-console
     console.warn('ChunkLoadError caught and ignored:', err.message);
     return false;
   }
-  // Let other errors fail the test as expected
-  return true;
-});
 
-// Ignore 'Unexpected token <' errors from webpack-dev-server fallback in E2E tests
-Cypress.on('uncaught:exception', (err) => {
+  // Ignore 'Unexpected token <' errors from webpack-dev-server fallback in E2E tests
   if (err.message.includes("Unexpected token '<'")) {
     return false;
   }
+
+  // Let all other errors (including timeout errors) fail the test as expected
   return true;
 });
 

frontend/src/__tests__/cypress/cypress/tests/e2e/README.md

@@ -143,6 +143,10 @@ npm run cypress:run \
 - `@Maintain`: Skip tests that require maintenance
 - `@NonConcurrent`: Skip tests that cannot run concurrently (to prevent resource conflicts)
 
+## Environment Variables
+
+For comprehensive documentation of all Cypress environment variables, see the [Cypress Environment Variables section in the main testing documentation](/docs/testing.md#cypress-environment-variables).
+
 ## Writing Tests
 
 e2e tests should focus on user journeys through the application, validating that features work as intended from the user's perspective.
@@ -198,7 +202,7 @@ Tests are parameterized using tags and applied to the 'it' block:
 * `Workbenches/Pipelines etc.`: Functional Area
 * `Destructive`: Tests that have the potential to break other tests (changing configuration etc.) 
 * `Bug`: Tests that are currently failing due to a Product Bug
-* `Maintain`: Tests that are currently failing and require maintenance 
+* `Maintain`: Tests that are currently failing and require maintenance
 * `NonConcurrent`: Tests that cannot run concurrently (to prevent resource conflicts)
 
 **Usage in tests:**
@@ -271,4 +275,3 @@ unset CY_TEST_CONFIG
 [Dashboard documentation]: docs/README.md
 [contributing guidelines]: CONTRIBUTING.md
 [issue]: https://github.com/opendatahub-io/odh-dashboard/issues/new/choose
-[definition of ready]: docs/definition-of-ready.md