feat(sync): RN-1545: Epic datatrak sync

.storybook/main.ts

@@ -48,12 +48,14 @@ const config: StorybookConfig = {
       resolve: {
         preserveSymlinks: true, // use the yarn workspace symlinks
         alias: {
-          http: path.resolve(__dirname, '../moduleMock.js'),
-          winston: path.resolve(__dirname, '../moduleMock.js'),
-          jsonwebtoken: path.resolve(__dirname, '../moduleMock.js'),
-          'node-fetch': path.resolve(__dirname, '../moduleMock.js'),
+          http: path.resolve(__dirname, '../mock/moduleMock.js'),
+          winston: path.resolve(__dirname, '../mock/moduleMock.js'),
+          jsonwebtoken: path.resolve(__dirname, '../mock/moduleMock.js'),
+          'node-fetch': path.resolve(__dirname, '../mock/moduleMock.js'),
           // This is a workaround for us using react-16 in the monorepo
           '@storybook/react-dom-shim': '@storybook/react-dom-shim/dist/react-16',
+          'pg-pubsub': path.resolve(__dirname, '../mock/moduleMock.js'),
+          '@node-rs/argon2': path.resolve(__dirname, '../mock/argon2ModuleMock.js'),
         },
       },
     });

README.md

@@ -80,6 +80,8 @@ While each package type has their own structure, there are a few common standard
 - [entity-server](/packages/entity-server/README.md)
 - [report-server](/packages/report-server/README.md)
 - [data-table-server](/packages/data-table-server/README.md)
+- [sync-server](/packages/sync-server/README.md)
+- [sync](/packages/sync/README.md)
 
 Server packages can be built by running `yarn workspace @tupaia/package-name build`. Server packages can then be started by running `yarn workspace @tupaia/package-name start`.
 

env/api-client.env.example

@@ -7,3 +7,4 @@ ENTITY_API_URL="http://localhost:8050/v1"
 REPORT_API_URL="http://localhost:8030/v1"
 TUPAIA_WEB_SERVER_API_URL="http://localhost:8100/v1"
 WEB_CONFIG_API_URL="http://localhost:8000/api/v1"
+SYNC_API_URL="http://localhost:8120/v1"

mock/argon2ModuleMock.js

@@ -0,0 +1,13 @@
+export const hash = () => {
+  throw new Error('@node-rs/argon2 not available in browser environment');
+};
+
+export const verify = () => {
+  throw new Error('@node-rs/argon2 not available in browser environment');
+};
+
+// Export any other functions your code imports
+export default {
+  hash,
+  verify,
+};

mock/pgMock.js

@@ -0,0 +1,8 @@
+export const types = () => {
+  throw new Error('pg not available in browser environment');
+};
+
+// Export any other functions your code imports
+export default {
+  types,
+};

mock/winston.js

@@ -0,0 +1,21 @@
+const winston = {
+  error: console.error,
+  warn: console.warn,
+  info: console.info,
+  log: console.log,
+  http: console.log,
+  verbose: console.log,
+  debug: console.debug,
+};
+
+// Support both CommonJS and ES modules
+export default winston;
+export const {
+  error,
+  warn,
+  info,
+  log,
+  http,
+  verbose,
+  debug,
+} = winston;

packages/access-policy/package.json

@@ -22,10 +22,14 @@
     "test:coverage": "yarn test --coverage"
   },
   "devDependencies": {
+    "@babel/core": "^7.26.10",
     "@babel/eslint-parser": "^7.27.5",
-    "@tupaia/utils": "workspace:*",
     "eslint": "^7.32.0",
     "npm-run-all": "^4.1.5",
     "rimraf": "^6.0.1"
+  },
+  "dependencies": {
+    "@tupaia/constants": "workspace:*",
+    "@tupaia/tsutils": "workspace:*"
   }
 }

packages/access-policy/src/AccessPolicy.js

@@ -112,9 +112,9 @@ export class AccessPolicy {
   /**
    * Return entities the user has access to the given permission group for
    *
-   * @param {string} [permissionGroup]
+   * @param {string} [permissionGroup] Permission group name
    *
-   * @returns entities[] The entity objects
+   * @returns {string[]} The entity codes
    */
   getEntitiesAllowed(permissionGroup) {
     const allEntityCodes = Object.keys(this.policy);

packages/access-policy/src/index.js

@@ -1 +1,18 @@
 export { AccessPolicy } from './AccessPolicy';
+export {
+  allowNoPermissions,
+  assertAdminPanelAccess,
+  assertAllPermissions,
+  assertAnyPermissions,
+  assertBESAdminAccess,
+  assertPermissionGroupAccess,
+  assertPermissionGroupsAccess,
+  assertVizBuilderAccess,
+  hasBESAdminAccess,
+  hasPermissionGroupAccess,
+  hasPermissionGroupsAccess,
+  hasSomePermissionGroupsAccess,
+  hasTupaiaAdminPanelAccess,
+  hasTupaiaAdminPanelAccessToCountry,
+  hasVizBuilderAccess,
+} from './permissions';

packages/access-policy/src/permissions.js

@@ -0,0 +1,144 @@
+import {
+  BES_ADMIN_PERMISSION_GROUP,
+  TUPAIA_ADMIN_PANEL_PERMISSION_GROUP,
+  VIZ_BUILDER_PERMISSION_GROUP,
+} from '@tupaia/constants';
+
+/**
+ * @typedef {import('./AccessPolicy').AccessPolicy} AccessPolicy
+ * @typedef {(accessPolicy: AccessPolicy) => true} SimplePermissionsAssertion
+ * @typedef {(accessPolicy: AccessPolicy) => boolean} SimplePermissionsChecker
+ * @typedef {string} PermissionGroupName
+ */
+
+/**
+ * Returns true all the time. This is for any route handlers that do not need permissions.
+ * @returns {true}
+ */
+export const allowNoPermissions = () => true;
+
+/**
+ * Returns true if all of the permissions assertions pass, or throws an error
+ * @param {function[]} assertions Each permissions assertion should return `true` or throw
+ * @param {string} errorMessage
+ * @returns {true}
+ * @privateRemarks Ideally, these should throw `PermissionsError`s but adding @tupaia/utils as a
+ * transitive dependency of @tupaia/meditrak-app causes MediTrak build to fail because it has
+ * imports from node:fs.
+ */
+export const assertAllPermissions = (assertions, errorMessage) => async accessPolicy => {
+  try {
+    for (const assertion of assertions) await assertion(accessPolicy);
+    return true;
+  } catch (e) {
+    throw new Error(errorMessage || e.message);
+  }
+};
+
+/**
+ * Returns `true` if any of the permissions assertions pass, otherwise throws
+ * @param {function[]} assertions Each permissions assertion should return true or throw
+ * @param {string} errorMessage
+ * @returns {true}
+ * @privateRemarks Ideally, these should throw `PermissionsError`s but adding @tupaia/utils as a
+ * transitive dependency of @tupaia/meditrak-app causes MediTrak build to fail because it has
+ * imports from node:fs.
+ */
+export const assertAnyPermissions = (assertions, errorMessage) => async accessPolicy => {
+  const combinedErrorMessages = ['One of the following conditions need to be satisfied:'];
+
+  for (const assertion of assertions) {
+    try {
+      await assertion(accessPolicy);
+      return true;
+    } catch (e) {
+      combinedErrorMessages.push(e.message);
+      // swallow specific errors, in case any assertion returns true
+    }
+  }
+  throw new Error(errorMessage || combinedErrorMessages.join('\n'));
+};
+
+/** @type {SimplePermissionsChecker} */
+export const hasBESAdminAccess = accessPolicy =>
+  accessPolicy.allowsSome(undefined, BES_ADMIN_PERMISSION_GROUP);
+
+/** @type {SimplePermissionsChecker} */
+export const hasVizBuilderAccess = accessPolicy =>
+  accessPolicy.allowsSome(undefined, VIZ_BUILDER_PERMISSION_GROUP);
+
+/**
+ * @param {AccessPolicy} accessPolicy
+ * @param {PermissionGroupName} permissionGroup
+ * @returns {boolean}
+ */
+export const hasPermissionGroupAccess = (accessPolicy, permissionGroup) =>
+  accessPolicy.allowsSome(undefined, permissionGroup);
+
+/**
+ * Has access to all permission groups inputted
+ * @param {AccessPolicy} accessPolicy
+ * @param {PermissionGroupName[]} permissionGroups
+ * @returns {boolean}
+ */
+export const hasPermissionGroupsAccess = (accessPolicy, permissionGroups) =>
+  permissionGroups.every(pg => accessPolicy.allowsSome(undefined, pg));
+
+/** @type {SimplePermissionsChecker} */
+export const hasSomePermissionGroupsAccess = (accessPolicy, permissionGroups) =>
+  permissionGroups.some(pg => accessPolicy.allowsSome(undefined, pg));
+
+/** @type {SimplePermissionsAssertion} */
+export const assertBESAdminAccess = accessPolicy => {
+  if (hasBESAdminAccess(accessPolicy)) return true;
+  throw new Error(`Need ${BES_ADMIN_PERMISSION_GROUP} access`);
+};
+
+/** @type {SimplePermissionsAssertion} */
+export const assertVizBuilderAccess = accessPolicy => {
+  if (hasVizBuilderAccess(accessPolicy)) return true;
+  throw new Error(`Need ${VIZ_BUILDER_PERMISSION_GROUP} access`);
+};
+
+/** @type {SimplePermissionsChecker} */
+export const hasTupaiaAdminPanelAccess = accessPolicy =>
+  accessPolicy.allowsSome(undefined, TUPAIA_ADMIN_PANEL_PERMISSION_GROUP);
+
+/**
+ * @param {AccessPolicy} accessPolicy
+ * @param {string} countryCode
+ * @returns {boolean}
+ */
+export const hasTupaiaAdminPanelAccessToCountry = (accessPolicy, countryCode) =>
+  accessPolicy.allows(countryCode, TUPAIA_ADMIN_PANEL_PERMISSION_GROUP);
+
+/** @type {SimplePermissionsAssertion} */
+export const assertAdminPanelAccess = accessPolicy => {
+  if (hasTupaiaAdminPanelAccess(accessPolicy)) return true;
+  throw new Error(`Need ${TUPAIA_ADMIN_PANEL_PERMISSION_GROUP} access`);
+};
+
+/**
+ * @param {AccessPolicy} accessPolicy
+ * @param {PermissionGroupName} permissionGroupName
+ * @returns {true}
+ */
+export const assertPermissionGroupAccess = (accessPolicy, permissionGroupName) => {
+  if (hasPermissionGroupAccess(accessPolicy, permissionGroupName)) return true;
+  throw new Error(`Need ${permissionGroupName} access`);
+};
+
+/**
+ * @param {AccessPolicy} accessPolicy
+ * @param {PermissionGroupName[]} permissionGroupNames
+ * @returns {true}
+ */
+export const assertPermissionGroupsAccess = (accessPolicy, permissionGroupNames) => {
+  if (
+    hasBESAdminAccess(accessPolicy) ||
+    hasPermissionGroupsAccess(accessPolicy, permissionGroupNames)
+  ) {
+    return true;
+  }
+  throw new Error(`Need access to ${permissionGroupNames.join(', ')}`);
+};

packages/admin-panel-server/src/app/createApp.ts

@@ -1,4 +1,5 @@
-import { NextFunction, Request, Response } from 'express';
+import { NextFunction, Request, RequestHandler, Response } from 'express';
+
 import { TupaiaDatabase } from '@tupaia/database';
 import {
   OrchestratorApiBuilder,
@@ -7,48 +8,49 @@ import {
   handleWith,
 } from '@tupaia/server-boilerplate';
 import { getEnvVarOrDefault } from '@tupaia/utils';
-import { AdminPanelSessionModel } from '../models';
-import { hasTupaiaAdminPanelAccess } from '../utils';
+
 import { upload } from '../middleware';
+import { AdminPanelSessionModel } from '../models';
 import {
   ExportDashboardVisualisationRequest,
   ExportDashboardVisualisationRoute,
   ExportDataTableRequest,
   ExportDataTableRoute,
+  ExportEntityHierarchiesRequest,
+  ExportEntityHierarchiesRoute,
   ExportMapOverlayVisualisationRequest,
   ExportMapOverlayVisualisationRoute,
   FetchDashboardVisualisationRequest,
   FetchDashboardVisualisationRoute,
+  FetchDataTableBuiltInParamsRequest,
+  FetchDataTableBuiltInParamsRoute,
+  FetchDataTablePreviewDataRequest,
+  FetchDataTablePreviewDataRoute,
   FetchHierarchyEntitiesRequest,
   FetchHierarchyEntitiesRoute,
   FetchMapOverlayVisualisationRequest,
   FetchMapOverlayVisualisationRoute,
   FetchReportPreviewDataRequest,
   FetchReportPreviewDataRoute,
-  FetchDataTablePreviewDataRequest,
-  FetchDataTablePreviewDataRoute,
+  FetchTransformSchemasRequest,
+  FetchTransformSchemasRoute,
   ImportDashboardVisualisationRequest,
   ImportDashboardVisualisationRoute,
   ImportDataTableRequest,
   ImportDataTableRoute,
+  ImportMapOverlayVisualisationRequest,
+  ImportMapOverlayVisualisationRoute,
+  PresentationOptionsPromptRequest,
+  PresentationOptionsPromptRoute,
   SaveDashboardVisualisationRequest,
   SaveDashboardVisualisationRoute,
   SaveMapOverlayVisualisationRequest,
   SaveMapOverlayVisualisationRoute,
   UploadTestDataRequest,
   UploadTestDataRoute,
   UserRoute,
-  ImportMapOverlayVisualisationRequest,
-  ImportMapOverlayVisualisationRoute,
-  FetchTransformSchemasRequest,
-  FetchTransformSchemasRoute,
-  FetchDataTableBuiltInParamsRequest,
-  FetchDataTableBuiltInParamsRoute,
-  ExportEntityHierarchiesRequest,
-  ExportEntityHierarchiesRoute,
-  PresentationOptionsPromptRequest,
-  PresentationOptionsPromptRoute,
 } from '../routes';
+import { hasTupaiaAdminPanelAccess } from '../utils';
 import { PromptManager } from '../viz-builder/prompts/PromptManager';
 
 export const addPromptManagerToContext =
@@ -131,22 +133,22 @@ export async function createApp(promptManager: PromptManager) {
     )
     .post<ImportDashboardVisualisationRequest>(
       'import/dashboardVisualisations',
-      upload.array('dashboardVisualisations'),
+      upload.array('dashboardVisualisations') as RequestHandler,
       handleWith(ImportDashboardVisualisationRoute),
     )
     .post<ImportDataTableRequest>(
       'import/dataTables',
-      upload.array('dataTables'),
+      upload.array('dataTables') as RequestHandler,
       handleWith(ImportDataTableRoute),
     )
     .post<ImportMapOverlayVisualisationRequest>(
       'import/mapOverlayVisualisations',
-      upload.array('mapOverlayVisualisations'),
+      upload.array('mapOverlayVisualisations') as RequestHandler,
       handleWith(ImportMapOverlayVisualisationRoute),
     )
     .post<UploadTestDataRequest>(
       'uploadTestData',
-      upload.single('testData'),
+      upload.single('testData') as RequestHandler,
       handleWith(UploadTestDataRoute),
     )
     .get<FetchMapOverlayVisualisationRequest>(

packages/admin-panel/package.json

@@ -44,7 +44,7 @@
     "ace-builds": "^1.10.1",
     "axios": "^1.12.2",
     "content-disposition-header": "^0.6.0",
-    "date-fns": "^2.29.2",
+    "date-fns": "^2.30.0",
     "file-saver": "^1.3.3",
     "jsoneditor": "^10.0.1",
     "lodash.debounce": "^4.0.8",

packages/admin-panel/src/pages/resources/ResourcePage.jsx

@@ -10,7 +10,6 @@ import { ResubmitSurveyResponseModal } from '../../surveyResponse/ResubmitSurvey
 import { Breadcrumbs } from '../../layout';
 import { useItemDetails } from '../../api/queries/useResourceDetails';
 import { ArchiveSurveyResponseModal } from '../../surveyResponse';
-import { useUser } from '../../api/queries';
 import { getExplodedFields, useHasBesAdminAccess, useHasVizBuilderAccess } from '../../utilities';
 
 const useEndpoint = (endpoint, details, params) => {

packages/aggregator/package.json

@@ -27,6 +27,7 @@
     "lodash.groupby": "^4.6.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.26.10",
     "@babel/eslint-parser": "^7.27.5",
     "@types/jest": "^29.5.14",
     "jest": "^29.7.0",