From 15cbf29d425889ec0acb9b4b05c6b5ca9332f954 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Tue, 28 Jan 2025 15:22:51 -0800 Subject: [PATCH 1/5] feat: experimental Glean event --- pnpm-lock.yaml | 202 ++++++++++++++---- servers/curated-corpus-api/.gitignore | 4 +- servers/curated-corpus-api/package.json | 10 +- .../src/events/eventHandlers.ts | 8 + .../src/events/glean/.gitignore | 1 + .../events/glean/ReviewedItemGleanHandler.ts | 111 ++++++++++ .../src/events/glean/backend-metrics.yaml | 99 +++++++++ servers/curated-corpus-api/src/main.ts | 4 +- 8 files changed, 390 insertions(+), 49 deletions(-) create mode 100644 servers/curated-corpus-api/src/events/glean/.gitignore create mode 100644 servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts create mode 100644 servers/curated-corpus-api/src/events/glean/backend-metrics.yaml diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 95b88671..42d8be0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -333,13 +333,13 @@ importers: version: link:../../packages/eslint-config-custom jest: specifier: 29.7.0 - version: 29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)) + version: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)) msw: specifier: 2.4.7 version: 2.4.7(typescript@5.6.2) ts-jest: specifier: 29.1.2 - version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)))(typescript@5.6.2) + version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)))(typescript@5.6.2) ts-patch: specifier: ^3.3.0 version: 3.3.0 @@ -480,22 +480,22 @@ importers: version: link:../eslint-config-custom jest: specifier: 29.7.0 - version: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + version: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) msw: specifier: 2.4.7 - version: 2.4.7(typescript@5.8.0-dev.20250205) + version: 2.4.7(typescript@5.8.0-dev.20250212) node-fetch: specifier: ^2.6.7 version: 2.7.0(encoding@0.1.13) ts-jest: specifier: 29.1.2 - version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(esbuild@0.23.1)(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)))(typescript@5.8.0-dev.20250205) + version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)))(typescript@5.8.0-dev.20250212) tsconfig: specifier: workspace:* version: link:../tsconfig tsup: specifier: 8.2.4 - version: 8.2.4(typescript@5.8.0-dev.20250205) + version: 8.2.4(typescript@5.8.0-dev.20250212) packages/eslint-config-custom: devDependencies: @@ -663,19 +663,19 @@ importers: version: link:../eslint-config-custom jest: specifier: 29.7.0 - version: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + version: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) msw: specifier: 2.4.7 - version: 2.4.7(typescript@5.8.0-dev.20250205) + version: 2.4.7(typescript@5.8.0-dev.20250212) ts-jest: specifier: 29.1.2 - version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(esbuild@0.23.1)(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)))(typescript@5.8.0-dev.20250205) + version: 29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)))(typescript@5.8.0-dev.20250212) tsconfig: specifier: workspace:* version: link:../tsconfig tsup: specifier: 8.2.4 - version: 8.2.4(typescript@5.8.0-dev.20250205) + version: 8.2.4(typescript@5.8.0-dev.20250212) packages/tsconfig: dependencies: @@ -842,6 +842,9 @@ importers: '@devoxa/prisma-relay-cursor-connection': specifier: 3.1.0 version: 3.1.0(@prisma/client@5.22.0(prisma@5.22.0)) + '@mozilla/glean': + specifier: ^5.0.3 + version: 5.0.3 '@pocket-tools/apollo-utils': specifier: 3.5.0 version: 3.5.0(encoding@0.1.13) @@ -905,6 +908,9 @@ importers: mime-types: specifier: 2.1.35 version: 2.1.35 + mozlog: + specifier: ^3.0.2 + version: 3.0.2 prisma: specifier: 5.22.0 version: 5.22.0 @@ -2083,6 +2089,11 @@ packages: '@microsoft/tsdoc@0.14.2': resolution: {integrity: sha512-9b8mPpKrfeGRuhFH5iO1iwCLeIIsV6+H1sRfxbkoGXIyQE2BTsPd9zqSqQJ+pv5sJ/hT5M1zvOFL02MnEezFug==} + '@mozilla/glean@5.0.3': + resolution: {integrity: sha512-zSCOOoFPC+W7rwwj9qPVMWnPwroHQkqNYe6SH9492RMPbDeWwxzqaeacX6ZmpPXopbcqfxQgyTET5Jbh1xLhHA==} + engines: {node: '>=12.20.0', npm: '>=7.0.0'} + hasBin: true + '@mswjs/interceptors@0.35.9': resolution: {integrity: sha512-SSnyl/4ni/2ViHKkiZb8eajA/eN1DNFaHjhGiLUdZvDz6PKF4COSf/17xqSz64nOo2Ia29SA6B2KNCsyCbVmaQ==} engines: {node: '>=18'} @@ -3701,6 +3712,10 @@ packages: resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} engines: {node: '>=8'} + ansi-regex@2.1.1: + resolution: {integrity: sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==} + engines: {node: '>=0.10.0'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -3709,6 +3724,10 @@ packages: resolution: {integrity: sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==} engines: {node: '>=12'} + ansi-styles@2.2.1: + resolution: {integrity: sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==} + engines: {node: '>=0.10.0'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -4021,6 +4040,10 @@ packages: resolution: {integrity: sha512-T2VJbcDuZQ0Tb2EWwSotMPJjgpy1/tGee1BTpUNsGZ/qgNjV2t7Mvu+d4600U564nbLesN1x2dPL+xii174Ekg==} engines: {node: '>=14.16'} + chalk@1.1.3: + resolution: {integrity: sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==} + engines: {node: '>=0.10.0'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -4313,6 +4336,9 @@ packages: resolution: {integrity: sha512-39BOQLs9ZjKh0/patS9nrT8wc3ioX3/eA/zgbKNopnF2wCqJEoxywwwElATYvRsXdnOxA/OQeQoFZ3rFjVajhg==} engines: {node: '>=4.0'} + dbug@0.4.2: + resolution: {integrity: sha512-nrmsMK1msY0WXwfA2czrKVDgpIYJR2JJaq5cX4DwW7Rxm11nXHqouh9wmubEs44bHYxk8CqeP/Jx4URqSB961w==} + debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -4925,6 +4951,9 @@ packages: fetch-retry@5.0.6: resolution: {integrity: sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==} + fflate@0.8.2: + resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -5203,6 +5232,10 @@ packages: resolution: {integrity: sha512-GGTKBX4SD7Wdb8mqeDLni2oaRGYQWjWHGKPQ24ZMnUtKfcsVoiv4uX8+LJr1K6U5VW2Lu1BwJnj7uiori0YtRw==} engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} + has-ansi@2.0.0: + resolution: {integrity: sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==} + engines: {node: '>=0.10.0'} + has-bigints@1.1.0: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} @@ -5400,6 +5433,10 @@ packages: resolution: {integrity: sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==} engines: {node: '>=12.0.0'} + intel@1.2.0: + resolution: {integrity: sha512-CUDyAtEeEeDo5YtwANOuDhxuFEOgInHvbMrBbhXCD4tAaHuzHM2llevtTeq2bmP8Jf7NkpN305pwDncRmhc1Wg==} + engines: {node: '>=4.0.0'} + internal-slot@1.1.0: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} @@ -6113,6 +6150,9 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + merge@2.1.1: + resolution: {integrity: sha512-jz+Cfrg9GWOZbQAnDQ4hlVnQky+341Yk5ru8bZSe6sIDTCIg8n9i/u7hSQGSVOF3C7lH6mGtqjkiT9G4wFLL0w==} + metadata-scraper@0.2.61: resolution: {integrity: sha512-ECV8r10nIVgn7Y5vY8lnlvi9vF1YgYBJjn2R1zrOcKRe47ra9Yg25ZE1ejL3Equqi8u2Mp346KHqIcR4PLdyTA==} @@ -6236,6 +6276,9 @@ packages: resolution: {integrity: sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==} engines: {node: '>= 0.8.0'} + mozlog@3.0.2: + resolution: {integrity: sha512-nu2pJV98gT0KFWE3sIHopR+QcSxZ2vCgnV+dvAAcCcOLjRoT3obtcINS4Vl0oc7zGEdhm0/MfE7D2MJGIwD/Ag==} + ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} @@ -7231,6 +7274,10 @@ packages: resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} engines: {node: '>=10.0.0'} + strftime@0.10.3: + resolution: {integrity: sha512-DZrDUeIF73eKJ4/GgGuv8UHWcUQPYDYfDeQFj3jrx+JZl6GQE656MbHIpvbo4mEG9a5DgS8GRCc5DxJXD2udDQ==} + engines: {node: '>=0.2.0'} + strict-event-emitter@0.5.1: resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==} @@ -7282,6 +7329,10 @@ packages: string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + strip-ansi@3.0.1: + resolution: {integrity: sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==} + engines: {node: '>=0.10.0'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -7331,6 +7382,10 @@ packages: resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==} engines: {node: '>=6.4.0'} + supports-color@2.0.0: + resolution: {integrity: sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==} + engines: {node: '>=0.8.0'} + supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} @@ -7351,6 +7406,9 @@ packages: resolution: {integrity: sha512-b19dMThMV4HVFynSAM1++gBHAbk2Tc/osgLIBZMKsyqh34jb2e8Os7T6ZW/Bt3pJFdBTd2JwAnAAEQV7rSNvcQ==} engines: {node: '>=0.10'} + symbol@0.3.1: + resolution: {integrity: sha512-SxMrE6uv9zhnBmTCpZna1u0TcZix1k2QASZ/DpF13rAo+0Ts40faFYsMTuAirgvbbjHw1byhJ949/fP20XzVZA==} + synckit@0.8.8: resolution: {integrity: sha512-HwOKAP7Wc5aRGYdKH+dw0PRRpbO841v2DENBtjnR5HFWoiNByAl7vrx3p0G/rCyYXQsrxqtX48TImFtPcIHSpQ==} engines: {node: ^14.18.0 || >=16.0.0} @@ -7650,8 +7708,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@5.8.0-dev.20250205: - resolution: {integrity: sha512-ElowVOyGLZ8RSwHA7UZF8M3n4iT/QaU6jCF9N9z2vT/lTjvM3K2eFliQsmF6Iaz9vOhFVUNfe1Mr8aXnpMXiHA==} + typescript@5.8.0-dev.20250212: + resolution: {integrity: sha512-KuSQZsJS5e7rBVbj7QY72K4kievEtZgBYyYYaxp0a3YmBlG5bpbuKJO0ltyRGkFcBgOVw4McWjKeL5+6MqRWOw==} engines: {node: '>=14.17'} hasBin: true @@ -7719,6 +7777,9 @@ packages: url-parse@1.5.10: resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + utcstring@0.1.0: + resolution: {integrity: sha512-1EpWQ6CECkoys7aX3LImrFo4nYIigY2RQHJTvgzZQCB4/oA6jJvTLTcgilTxX57GrSHDIVMtGwYd+SujGJvvyw==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -9590,7 +9651,7 @@ snapshots: - supports-color - ts-node - '@jest/core@29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205))': + '@jest/core@29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212))': dependencies: '@jest/console': 29.7.0 '@jest/reporters': 29.7.0 @@ -9604,7 +9665,7 @@ snapshots: exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + jest-config: 29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -9790,6 +9851,12 @@ snapshots: '@microsoft/tsdoc@0.14.2': {} + '@mozilla/glean@5.0.3': + dependencies: + fflate: 0.8.2 + tslib: 2.7.0 + uuid: 9.0.1 + '@mswjs/interceptors@0.35.9': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -12037,10 +12104,14 @@ snapshots: dependencies: type-fest: 0.21.3 + ansi-regex@2.1.1: {} + ansi-regex@5.0.1: {} ansi-regex@6.1.0: {} + ansi-styles@2.2.1: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -12469,6 +12540,14 @@ snapshots: dependencies: chalk: 5.3.0 + chalk@1.1.3: + dependencies: + ansi-styles: 2.2.1 + escape-string-regexp: 1.0.5 + has-ansi: 2.0.0 + strip-ansi: 3.0.1 + supports-color: 2.0.0 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -12717,13 +12796,13 @@ snapshots: - supports-color - ts-node - create-jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)): + create-jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)): dependencies: '@jest/types': 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + jest-config: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -12786,6 +12865,8 @@ snapshots: date-format@4.0.14: {} + dbug@0.4.2: {} + debug@2.6.9: dependencies: ms: 2.0.0 @@ -12927,7 +13008,7 @@ snapshots: dependencies: semver: 7.7.0 shelljs: 0.8.5 - typescript: 5.8.0-dev.20250205 + typescript: 5.8.0-dev.20250212 drange@1.1.1: {} @@ -13567,6 +13648,8 @@ snapshots: fetch-retry@5.0.6: {} + fflate@0.8.2: {} + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -13887,6 +13970,10 @@ snapshots: graphql@16.9.0: {} + has-ansi@2.0.0: + dependencies: + ansi-regex: 2.1.1 + has-bigints@1.1.0: {} has-flag@3.0.0: {} @@ -14104,6 +14191,15 @@ snapshots: through: 2.3.8 wrap-ansi: 6.2.0 + intel@1.2.0: + dependencies: + chalk: 1.1.3 + dbug: 0.4.2 + stack-trace: 0.0.10 + strftime: 0.10.3 + symbol: 0.3.1 + utcstring: 0.1.0 + internal-slot@1.1.0: dependencies: es-errors: 1.3.0 @@ -14439,16 +14535,16 @@ snapshots: - supports-color - ts-node - jest-cli@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)): + jest-cli@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)): dependencies: - '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + create-jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) exit: 0.1.2 import-local: 3.2.0 - jest-config: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + jest-config: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -14520,7 +14616,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)): + jest-config@29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)): dependencies: '@babel/core': 7.26.7 '@jest/test-sequencer': 29.7.0 @@ -14546,7 +14642,7 @@ snapshots: strip-json-comments: 3.1.1 optionalDependencies: '@types/node': 20.12.14 - ts-node: 10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205) + ts-node: 10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212) transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -14582,7 +14678,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)): + jest-config@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)): dependencies: '@babel/core': 7.26.7 '@jest/test-sequencer': 29.7.0 @@ -14608,7 +14704,7 @@ snapshots: strip-json-comments: 3.1.1 optionalDependencies: '@types/node': 22.12.0 - ts-node: 10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205) + ts-node: 10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212) transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -14852,12 +14948,12 @@ snapshots: - supports-color - ts-node - jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)): + jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)): dependencies: - '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + '@jest/core': 29.7.0(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) '@jest/types': 29.6.3 import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + jest-cli: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -15213,6 +15309,8 @@ snapshots: merge2@1.4.1: {} + merge@2.1.1: {} + metadata-scraper@0.2.61: dependencies: domino: 2.1.6 @@ -15322,6 +15420,11 @@ snapshots: transitivePeerDependencies: - supports-color + mozlog@3.0.2: + dependencies: + intel: 1.2.0 + merge: 2.1.1 + ms@2.0.0: {} ms@2.1.3: {} @@ -15348,7 +15451,7 @@ snapshots: optionalDependencies: typescript: 5.6.2 - msw@2.4.7(typescript@5.8.0-dev.20250205): + msw@2.4.7(typescript@5.8.0-dev.20250212): dependencies: '@bundled-es-modules/cookie': 2.0.1 '@bundled-es-modules/statuses': 1.0.1 @@ -15368,7 +15471,7 @@ snapshots: type-fest: 4.33.0 yargs: 17.7.2 optionalDependencies: - typescript: 5.8.0-dev.20250205 + typescript: 5.8.0-dev.20250212 murmurhash3js@3.0.1: {} @@ -16398,6 +16501,8 @@ snapshots: streamsearch@1.1.0: {} + strftime@0.10.3: {} + strict-event-emitter@0.5.1: {} string-length@4.0.2: @@ -16483,6 +16588,10 @@ snapshots: dependencies: safe-buffer: 5.2.1 + strip-ansi@3.0.1: + dependencies: + ansi-regex: 2.1.1 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 @@ -16539,6 +16648,8 @@ snapshots: transitivePeerDependencies: - supports-color + supports-color@2.0.0: {} + supports-color@5.5.0: dependencies: has-flag: 3.0.0 @@ -16555,6 +16666,8 @@ snapshots: symbol-observable@4.0.0: {} + symbol@0.3.1: {} + synckit@0.8.8: dependencies: '@pkgr/core': 0.1.1 @@ -16690,29 +16803,28 @@ snapshots: dependencies: tslib: 2.7.0 - ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(esbuild@0.23.1)(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)))(typescript@5.8.0-dev.20250205): + ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)))(typescript@5.6.2): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205)) + jest: 29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)) jest-util: 29.7.0 json5: 2.2.3 lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.7.0 - typescript: 5.8.0-dev.20250205 + typescript: 5.6.2 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 7.26.7 '@jest/types': 29.6.3 babel-jest: 29.7.0(@babel/core@7.26.7) - esbuild: 0.23.1 - ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)))(typescript@5.6.2): + ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)))(typescript@5.6.2): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@20.12.14)(ts-node@10.9.2(@types/node@20.12.14)(typescript@5.6.2)) + jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)) jest-util: 29.7.0 json5: 2.2.3 lodash.memoize: 4.1.2 @@ -16725,17 +16837,17 @@ snapshots: '@jest/types': 29.6.3 babel-jest: 29.7.0(@babel/core@7.26.7) - ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)))(typescript@5.6.2): + ts-jest@29.1.2(@babel/core@7.26.7)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.7))(jest@29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)))(typescript@5.8.0-dev.20250212): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 - jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.6.2)) + jest: 29.7.0(@types/node@22.12.0)(ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212)) jest-util: 29.7.0 json5: 2.2.3 lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.7.0 - typescript: 5.6.2 + typescript: 5.8.0-dev.20250212 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 7.26.7 @@ -16779,7 +16891,7 @@ snapshots: yn: 3.1.1 optional: true - ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250205): + ts-node@10.9.2(@types/node@22.12.0)(typescript@5.8.0-dev.20250212): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 @@ -16793,7 +16905,7 @@ snapshots: create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.8.0-dev.20250205 + typescript: 5.8.0-dev.20250212 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 optional: true @@ -16848,7 +16960,7 @@ snapshots: - tsx - yaml - tsup@8.2.4(typescript@5.8.0-dev.20250205): + tsup@8.2.4(typescript@5.8.0-dev.20250212): dependencies: bundle-require: 5.1.0(esbuild@0.23.1) cac: 6.7.14 @@ -16867,7 +16979,7 @@ snapshots: sucrase: 3.35.0 tree-kill: 1.2.2 optionalDependencies: - typescript: 5.8.0-dev.20250205 + typescript: 5.8.0-dev.20250212 transitivePeerDependencies: - jiti - supports-color @@ -16972,7 +17084,7 @@ snapshots: typescript@5.6.2: {} - typescript@5.8.0-dev.20250205: {} + typescript@5.8.0-dev.20250212: {} typia@7.6.0(@samchon/openapi@2.4.1)(typescript@5.6.2): dependencies: @@ -17041,6 +17153,8 @@ snapshots: querystringify: 2.2.0 requires-port: 1.0.0 + utcstring@0.1.0: {} + util-deprecate@1.0.2: {} utils-merge@1.0.1: {} diff --git a/servers/curated-corpus-api/.gitignore b/servers/curated-corpus-api/.gitignore index e6ded1e8..6bd7a33e 100644 --- a/servers/curated-corpus-api/.gitignore +++ b/servers/curated-corpus-api/.gitignore @@ -1,3 +1,5 @@ # generated by docker/in CI, but should not be in in source control schema-admin-api.graphql -schema-client-api.graphql \ No newline at end of file +schema-client-api.graphql + +.venv/ diff --git a/servers/curated-corpus-api/package.json b/servers/curated-corpus-api/package.json index e935cccb..c38fabfe 100644 --- a/servers/curated-corpus-api/package.json +++ b/servers/curated-corpus-api/package.json @@ -4,9 +4,11 @@ "description": "", "main": "dist/main.js", "scripts": { - "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public", + "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public && npm run build-glean-debug", "build-schema-admin": "node dist/admin/buildSchema.js", "build-schema-public": "node dist/public/buildSchema.js", + "build-glean-debug": "glean translate src/events/glean/backend-metrics.yaml -f typescript -o ./src/events/glean/generated", + "build-glean": "glean translate src/events/glean/backend-metrics.yaml -f typescript_server --option module_spec=es -o ./src/events/glean/generated", "watch": "tsc -w & nodemon", "start": "npm run migrate:prisma-deploy && node dist/main.js", "dev": "npm run migrate:prisma-deploy && npm run build && npm run watch", @@ -31,13 +33,14 @@ "author": "", "license": "ISC", "dependencies": { - "@apollo/server-plugin-response-cache": "4.1.3", "@apollo/server": "4.11.0", + "@apollo/server-plugin-response-cache": "4.1.3", "@apollo/subgraph": "2.9.0", "@aws-sdk/client-eventbridge": "3.529.1", "@aws-sdk/client-s3": "3.529.1", "@aws-sdk/lib-storage": "3.529.1", "@devoxa/prisma-relay-cursor-connection": "3.1.0", + "@mozilla/glean": "^5.0.3", "@pocket-tools/apollo-utils": "3.5.0", "@pocket-tools/feature-flags-client": "1.0.0", "@pocket-tools/sentry": "1.0.0", @@ -52,13 +55,14 @@ "dataloader": "^2.2.2", "date-fns": "2.29.3", "express": "4.19.2", + "graphql": "16.9.0", "graphql-scalars": "1.22.4", "graphql-tag": "2.12.6", "graphql-upload": "15.0.2", - "graphql": "16.9.0", "luxon": "3.4.4", "metadata-scraper": "^0.2.61", "mime-types": "2.1.35", + "mozlog": "^3.0.2", "prisma": "5.22.0", "tslib": "2.7.0", "uuid": "9.0.1" diff --git a/servers/curated-corpus-api/src/events/eventHandlers.ts b/servers/curated-corpus-api/src/events/eventHandlers.ts index 2581158c..69555618 100644 --- a/servers/curated-corpus-api/src/events/eventHandlers.ts +++ b/servers/curated-corpus-api/src/events/eventHandlers.ts @@ -4,6 +4,7 @@ import config from '../config'; import { ReviewedItemSnowplowHandler } from './snowplow/ReviewedItemSnowplowHandler'; import { ScheduledItemSnowplowHandler } from './snowplow/ScheduledItemSnowplowHandler'; import { EventBusHandler } from './eventBus'; +import { ReviewedItemGleanHandler } from './glean/ReviewedItemGleanHandler'; export type CuratedCorpusEventHandlerFn = ( emitter: CuratedCorpusEventEmitter @@ -42,3 +43,10 @@ export function corpusScheduleSnowplowEventHandler( export function eventBusHandler(emitter: CuratedCorpusEventEmitter): void { new EventBusHandler(emitter); } + +export function corpusItemGleanEventHandler(emitter: CuratedCorpusEventEmitter): void { + const gleanEventsToListen = Object.values( + config.snowplow.corpusItemEvents + ) as string[]; + new ReviewedItemGleanHandler(emitter, gleanEventsToListen); +} diff --git a/servers/curated-corpus-api/src/events/glean/.gitignore b/servers/curated-corpus-api/src/events/glean/.gitignore new file mode 100644 index 00000000..35fcc150 --- /dev/null +++ b/servers/curated-corpus-api/src/events/glean/.gitignore @@ -0,0 +1 @@ +generated/ diff --git a/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts new file mode 100644 index 00000000..a256add1 --- /dev/null +++ b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts @@ -0,0 +1,111 @@ +import { CuratedCorpusEventEmitter } from '../curatedCorpusEventEmitter'; +import { serverLogger } from '@pocket-tools/ts-logger'; +import * as Sentry from '@sentry/node'; +import { createEventsServerEventLogger } from './generated/server_events'; + +import { + ApprovedCorpusItemPayload, + BaseEventData, + RejectedCorpusItemPayload, + ReviewedCorpusItemPayload, +} from '../types'; +import { CuratedStatus } from '.prisma/client'; +import { getUnixTimestamp } from '../../shared/utils'; +import { CorpusItemSource } from 'content-common'; +import { CorpusReviewStatus } from '../snowplow/schema'; + +const gleanLogger = createEventsServerEventLogger({ + applicationId: 'curated-corpus-api', + appDisplayVersion: '1.0.0', + channel: 'production', + logger_options: { app: 'glean-logger' }, +}); + +export class ReviewedItemGleanHandler { + constructor( + protected emitter: CuratedCorpusEventEmitter, + events: string[] + ) { + events.forEach((eventName) => { + this.emitter.on(eventName, (data) => this.process(data)); + }); + } + + async process(data: ReviewedCorpusItemPayload & BaseEventData) { + try { + const gleanExtras = ReviewedItemGleanHandler.buildGleanExtras(data); + + gleanLogger.recordCuratedCorpusReviewedCorpusItem({ + user_agent: 'unknown', + ip_address: 'unknown', + ...gleanExtras, + experimental_json: gleanExtras.experimental_json ?? '', + }); + } catch (ex) { + serverLogger.error('ReviewedItemGleanHandler: failed to record Glean event', { + error: ex, + eventType: data.eventType, + }); + Sentry.captureException(ex); + } + } + + private static buildGleanExtras(data: ReviewedCorpusItemPayload & BaseEventData) { + const item = data.reviewedCorpusItem; + + let isApproved = false; + let corpusReviewStatus: string; + let approvedExternalId = ''; + let rejectedExternalId = ''; + let rejectionReasons = ''; + + if ((item as ApprovedCorpusItemPayload).status !== undefined) { + const status = (item as ApprovedCorpusItemPayload).status; + corpusReviewStatus = + status === CuratedStatus.RECOMMENDATION + ? CorpusReviewStatus.RECOMMENDATION + : CorpusReviewStatus.CORPUS; + isApproved = true; + approvedExternalId = (item as ApprovedCorpusItemPayload).externalId; + } else { + corpusReviewStatus = CorpusReviewStatus.REJECTED; + rejectedExternalId = (item as RejectedCorpusItemPayload).externalId; + if ((item as RejectedCorpusItemPayload).reason) { + rejectionReasons = JSON.stringify( + (item as RejectedCorpusItemPayload).reason.split(',') + ); + } + } + + return { + object_version: 'new', + approved_corpus_item_external_id: isApproved ? approvedExternalId : '', + rejected_corpus_item_external_id: isApproved ? '' : rejectedExternalId, + prospect_id: item.prospectId ?? '', + url: item.url ?? '', + loaded_from: '', // TODO: Property 'source' does not exist on type 'ApprovedCorpusItemPayload | RejectedCorpusItemPayload' + corpus_review_status: corpusReviewStatus, + rejection_reasons_json: rejectionReasons, + action_screen: (item as any).action_screen ?? '', + title: item.title ?? '', + excerpt: isApproved ? (item as ApprovedCorpusItemPayload).excerpt ?? '' : '', + image_url: isApproved ? (item as ApprovedCorpusItemPayload).imageUrl ?? '' : '', + language: item.language ?? '', + topic: item.topic ?? '', + is_collection: isApproved ? !!(item as ApprovedCorpusItemPayload).isCollection : false, + is_syndicated: isApproved ? !!(item as ApprovedCorpusItemPayload).isSyndicated : false, + is_time_sensitive: isApproved ? !!(item as ApprovedCorpusItemPayload).isTimeSensitive : false, + created_at: item.createdAt ? getUnixTimestamp(item.createdAt).toString() : '', + created_by: item.createdBy ?? '', + updated_at: isApproved && (item as ApprovedCorpusItemPayload).updatedAt + ? getUnixTimestamp((item as ApprovedCorpusItemPayload).updatedAt).toString() + : '', + updated_by: isApproved ? (item as ApprovedCorpusItemPayload).updatedBy ?? '' : '', + authors_json: isApproved && (item as ApprovedCorpusItemPayload).authors + ? JSON.stringify((item as ApprovedCorpusItemPayload).authors.map((a) => a.name)) + : '', + publisher: isApproved ? (item as ApprovedCorpusItemPayload).publisher ?? '' : '', + experimental_json: '', + }; + } +} diff --git a/servers/curated-corpus-api/src/events/glean/backend-metrics.yaml b/servers/curated-corpus-api/src/events/glean/backend-metrics.yaml new file mode 100644 index 00000000..a13d27f7 --- /dev/null +++ b/servers/curated-corpus-api/src/events/glean/backend-metrics.yaml @@ -0,0 +1,99 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# For more information on configuring this file: +# https://mozilla.github.io/glean/book/reference/yaml/metrics.html + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 + +curated_corpus: + reviewed_corpus_item: + type: event + description: > + Recorded when a curated corpus item is reviewed. This is converted from the Snowplow + "reviewed_corpus_item" event. Any array or object fields are JSON-encoded into strings. + bugs: + - https://bugzilla.mozilla.org/show_bug.cgi?id=000000 + data_reviews: + - https://bugzilla.mozilla.org/show_bug.cgi?id=000000 + notification_emails: + - your-team@example.com + expires: never + # Glean events always require a 'ping' lifetime + send_in_pings: + - events + extra_keys: + object_version: + type: string + description: "Indicates old/new version." + approved_corpus_item_external_id: + type: string + description: "A guid that identifies the reviewed_corpus_item’s approved_corpus_item_external_id in backend systems, sometimes referred to as an approved corpus item’s external_id." + rejected_corpus_item_external_id: + type: string + description: "A guid that identifies the reviewed_corpus_item’s rejected_corpus_item_external_id in backend systems, sometimes referred to as a rejected corpus item’s external_id." + prospect_id: + type: string + description: "Identifier for the curation prospect." + url: + type: string + description: "URL of the reviewed corpus item." + loaded_from: + type: string + description: "Backend source for the item (PROSPECT, MANUAL, etc.)." + corpus_review_status: + type: string + description: "Decision by the curator on the item’s validity (recommendation/corpus/rejected)." + rejection_reasons_json: + type: string + description: "JSON-encoded array of rejection reasons." + action_screen: + type: string + description: "UI screen where action took place (PROSPECTING, SCHEDULE, CORPUS) or null." + title: + type: string + description: "Title of the reviewed corpus item." + excerpt: + type: string + description: "Excerpt for the reviewed corpus item." + image_url: + type: string + description: "URL of the main image." + language: + type: string + description: "Language code of the item." + topic: + type: string + description: "Topic of the item." + is_collection: + type: boolean + description: "Whether this item is a collection." + is_syndicated: + type: boolean + description: "Whether the item is syndicated." + is_time_sensitive: + type: boolean + description: "Whether the item is time-sensitive (e.g. news)." + created_at: + type: string + description: "Unix timestamp (in seconds) for item creation." + created_by: + type: string + description: "Curator who created the item." + updated_at: + type: string + description: "Unix timestamp (in seconds) for last item update." + updated_by: + type: string + description: "Curator who last updated the item." + authors_json: + type: string + description: "JSON-encoded list of authors." + publisher: + type: string + description: "Online publication that published this story." + experimental_json: + type: string + description: "Experimental metadata properties associated with the reviewed Corpus Item." diff --git a/servers/curated-corpus-api/src/main.ts b/servers/curated-corpus-api/src/main.ts index 0ef1d76b..94cbcb0a 100644 --- a/servers/curated-corpus-api/src/main.ts +++ b/servers/curated-corpus-api/src/main.ts @@ -39,9 +39,10 @@ import { } from './events/init'; import { + corpusItemGleanEventHandler, corpusItemSnowplowEventHandler, corpusScheduleSnowplowEventHandler, - eventBusHandler, + eventBusHandler } from './events/eventHandlers'; // Initialize event handlers, this is outside server setup as tests @@ -50,4 +51,5 @@ initItemEventHandlers(curatedCorpusEventEmitter, [ corpusItemSnowplowEventHandler, corpusScheduleSnowplowEventHandler, eventBusHandler, + corpusItemGleanEventHandler, ]); From e59c44a846b9aa53bc7728242a402009d0436c4d Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Tue, 28 Jan 2025 15:53:05 -0800 Subject: [PATCH 2/5] fix package.json --- servers/curated-corpus-api/package.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/servers/curated-corpus-api/package.json b/servers/curated-corpus-api/package.json index c38fabfe..2d46a514 100644 --- a/servers/curated-corpus-api/package.json +++ b/servers/curated-corpus-api/package.json @@ -4,10 +4,9 @@ "description": "", "main": "dist/main.js", "scripts": { - "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public && npm run build-glean-debug", + "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public && npm run build-glean", "build-schema-admin": "node dist/admin/buildSchema.js", "build-schema-public": "node dist/public/buildSchema.js", - "build-glean-debug": "glean translate src/events/glean/backend-metrics.yaml -f typescript -o ./src/events/glean/generated", "build-glean": "glean translate src/events/glean/backend-metrics.yaml -f typescript_server --option module_spec=es -o ./src/events/glean/generated", "watch": "tsc -w & nodemon", "start": "npm run migrate:prisma-deploy && node dist/main.js", @@ -33,8 +32,8 @@ "author": "", "license": "ISC", "dependencies": { - "@apollo/server": "4.11.0", "@apollo/server-plugin-response-cache": "4.1.3", + "@apollo/server": "4.11.0", "@apollo/subgraph": "2.9.0", "@aws-sdk/client-eventbridge": "3.529.1", "@aws-sdk/client-s3": "3.529.1", @@ -55,10 +54,10 @@ "dataloader": "^2.2.2", "date-fns": "2.29.3", "express": "4.19.2", - "graphql": "16.9.0", "graphql-scalars": "1.22.4", "graphql-tag": "2.12.6", "graphql-upload": "15.0.2", + "graphql": "16.9.0", "luxon": "3.4.4", "metadata-scraper": "^0.2.61", "mime-types": "2.1.35", From 4750b4cbd34eb5d5d22f07cd0d67c02c390ed890 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Fri, 14 Feb 2025 17:48:40 -0800 Subject: [PATCH 3/5] use serverLogger as Glean stream --- servers/curated-corpus-api/package.json | 8 +- .../events/glean/ReviewedItemGleanHandler.ts | 260 ++++++++++-------- 2 files changed, 153 insertions(+), 115 deletions(-) diff --git a/servers/curated-corpus-api/package.json b/servers/curated-corpus-api/package.json index 2d46a514..253b2700 100644 --- a/servers/curated-corpus-api/package.json +++ b/servers/curated-corpus-api/package.json @@ -4,10 +4,9 @@ "description": "", "main": "dist/main.js", "scripts": { - "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public && npm run build-glean", + "build": "rm -rf dist && tsc && npm run build-schema-admin && npm run build-schema-public", "build-schema-admin": "node dist/admin/buildSchema.js", "build-schema-public": "node dist/public/buildSchema.js", - "build-glean": "glean translate src/events/glean/backend-metrics.yaml -f typescript_server --option module_spec=es -o ./src/events/glean/generated", "watch": "tsc -w & nodemon", "start": "npm run migrate:prisma-deploy && node dist/main.js", "dev": "npm run migrate:prisma-deploy && npm run build && npm run watch", @@ -22,8 +21,9 @@ "migrate:prisma-dev": "prisma migrate dev", "migrate:prisma-deploy": "prisma migrate deploy", "migrate:prisma-reset": "prisma migrate reset --skip-seed --force", - "prebuild": "dotenv -e .env.ci -- prisma generate", - "pretest": "dotenv -e .env.ci -- prisma generate", + "generate-glean": "glean translate src/events/glean/backend-metrics.yaml -f typescript_server --option module_spec=es -o ./src/events/glean/generated", + "prebuild": "dotenv -e .env.ci -- prisma generate && npm run generate-glean", + "pretest": "dotenv -e .env.ci -- prisma generate && npm run generate-glean", "pretest-integrations": "dotenv -e .env.ci -- prisma migrate reset --skip-seed --force" }, "prisma": { diff --git a/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts index a256add1..1a6d015c 100644 --- a/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts +++ b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts @@ -1,111 +1,149 @@ -import { CuratedCorpusEventEmitter } from '../curatedCorpusEventEmitter'; -import { serverLogger } from '@pocket-tools/ts-logger'; -import * as Sentry from '@sentry/node'; -import { createEventsServerEventLogger } from './generated/server_events'; - -import { - ApprovedCorpusItemPayload, - BaseEventData, - RejectedCorpusItemPayload, - ReviewedCorpusItemPayload, -} from '../types'; -import { CuratedStatus } from '.prisma/client'; -import { getUnixTimestamp } from '../../shared/utils'; -import { CorpusItemSource } from 'content-common'; -import { CorpusReviewStatus } from '../snowplow/schema'; - -const gleanLogger = createEventsServerEventLogger({ - applicationId: 'curated-corpus-api', - appDisplayVersion: '1.0.0', - channel: 'production', - logger_options: { app: 'glean-logger' }, -}); - -export class ReviewedItemGleanHandler { - constructor( - protected emitter: CuratedCorpusEventEmitter, - events: string[] - ) { - events.forEach((eventName) => { - this.emitter.on(eventName, (data) => this.process(data)); - }); - } - - async process(data: ReviewedCorpusItemPayload & BaseEventData) { - try { - const gleanExtras = ReviewedItemGleanHandler.buildGleanExtras(data); - - gleanLogger.recordCuratedCorpusReviewedCorpusItem({ - user_agent: 'unknown', - ip_address: 'unknown', - ...gleanExtras, - experimental_json: gleanExtras.experimental_json ?? '', - }); - } catch (ex) { - serverLogger.error('ReviewedItemGleanHandler: failed to record Glean event', { - error: ex, - eventType: data.eventType, - }); - Sentry.captureException(ex); - } - } - - private static buildGleanExtras(data: ReviewedCorpusItemPayload & BaseEventData) { - const item = data.reviewedCorpusItem; - - let isApproved = false; - let corpusReviewStatus: string; - let approvedExternalId = ''; - let rejectedExternalId = ''; - let rejectionReasons = ''; - - if ((item as ApprovedCorpusItemPayload).status !== undefined) { - const status = (item as ApprovedCorpusItemPayload).status; - corpusReviewStatus = - status === CuratedStatus.RECOMMENDATION - ? CorpusReviewStatus.RECOMMENDATION - : CorpusReviewStatus.CORPUS; - isApproved = true; - approvedExternalId = (item as ApprovedCorpusItemPayload).externalId; - } else { - corpusReviewStatus = CorpusReviewStatus.REJECTED; - rejectedExternalId = (item as RejectedCorpusItemPayload).externalId; - if ((item as RejectedCorpusItemPayload).reason) { - rejectionReasons = JSON.stringify( - (item as RejectedCorpusItemPayload).reason.split(',') - ); - } - } - - return { - object_version: 'new', - approved_corpus_item_external_id: isApproved ? approvedExternalId : '', - rejected_corpus_item_external_id: isApproved ? '' : rejectedExternalId, - prospect_id: item.prospectId ?? '', - url: item.url ?? '', - loaded_from: '', // TODO: Property 'source' does not exist on type 'ApprovedCorpusItemPayload | RejectedCorpusItemPayload' - corpus_review_status: corpusReviewStatus, - rejection_reasons_json: rejectionReasons, - action_screen: (item as any).action_screen ?? '', - title: item.title ?? '', - excerpt: isApproved ? (item as ApprovedCorpusItemPayload).excerpt ?? '' : '', - image_url: isApproved ? (item as ApprovedCorpusItemPayload).imageUrl ?? '' : '', - language: item.language ?? '', - topic: item.topic ?? '', - is_collection: isApproved ? !!(item as ApprovedCorpusItemPayload).isCollection : false, - is_syndicated: isApproved ? !!(item as ApprovedCorpusItemPayload).isSyndicated : false, - is_time_sensitive: isApproved ? !!(item as ApprovedCorpusItemPayload).isTimeSensitive : false, - created_at: item.createdAt ? getUnixTimestamp(item.createdAt).toString() : '', - created_by: item.createdBy ?? '', - updated_at: isApproved && (item as ApprovedCorpusItemPayload).updatedAt - ? getUnixTimestamp((item as ApprovedCorpusItemPayload).updatedAt).toString() - : '', - updated_by: isApproved ? (item as ApprovedCorpusItemPayload).updatedBy ?? '' : '', - authors_json: isApproved && (item as ApprovedCorpusItemPayload).authors - ? JSON.stringify((item as ApprovedCorpusItemPayload).authors.map((a) => a.name)) - : '', - publisher: isApproved ? (item as ApprovedCorpusItemPayload).publisher ?? '' : '', - experimental_json: '', - }; - } -} +import { CuratedCorpusEventEmitter } from '../curatedCorpusEventEmitter'; +import { serverLogger } from '@pocket-tools/ts-logger'; +import * as Sentry from '@sentry/node'; +import { createEventsServerEventLogger } from './generated/server_events'; + +import { + ApprovedCorpusItemPayload, + BaseEventData, + RejectedCorpusItemPayload, + ReviewedCorpusItemPayload, +} from '../types'; +import { CuratedStatus } from '.prisma/client'; +import { getUnixTimestamp } from '../../shared/utils'; +import { CorpusReviewStatus } from '../snowplow/schema'; + +// Create a custom stream that forwards messages to serverLogger. +const customStream = { + write: (message: string) => { + // Forward the log message to the existing serverLogger. + serverLogger.info(message); + }, +}; + +// We know that mozlog supports the `stream` property, so we assert our object to `any` to suppress TS error. +const gleanLogger = createEventsServerEventLogger({ + applicationId: 'curated-corpus-api', + appDisplayVersion: '1.0.0', + channel: 'production', + logger_options: { + app: 'curated-corpus-api', + stream: customStream, // Use our custom stream for log output. + } as any, // generated Glean code does not define stream logger option that mozlog supports +}); + +export class ReviewedItemGleanHandler { + constructor( + protected emitter: CuratedCorpusEventEmitter, + events: string[], + ) { + events.forEach((eventName) => { + this.emitter.on(eventName, (data) => this.process(data)); + }); + } + + async process(data: ReviewedCorpusItemPayload & BaseEventData) { + try { + const gleanExtras = ReviewedItemGleanHandler.buildGleanExtras(data); + + gleanLogger.recordCuratedCorpusReviewedCorpusItem({ + user_agent: 'unknown', + ip_address: 'unknown', + ...gleanExtras, + experimental_json: gleanExtras.experimental_json ?? '', + }); + } catch (ex) { + serverLogger.error( + 'ReviewedItemGleanHandler: failed to record Glean event', + { + error: ex, + eventType: data.eventType, + }, + ); + Sentry.captureException(ex); + } + } + + private static buildGleanExtras( + data: ReviewedCorpusItemPayload & BaseEventData, + ) { + const item = data.reviewedCorpusItem; + + let isApproved = false; + let corpusReviewStatus: string; + let approvedExternalId = ''; + let rejectedExternalId = ''; + let rejectionReasons = ''; + + if ((item as ApprovedCorpusItemPayload).status !== undefined) { + const status = (item as ApprovedCorpusItemPayload).status; + corpusReviewStatus = + status === CuratedStatus.RECOMMENDATION + ? CorpusReviewStatus.RECOMMENDATION + : CorpusReviewStatus.CORPUS; + isApproved = true; + approvedExternalId = (item as ApprovedCorpusItemPayload).externalId; + } else { + corpusReviewStatus = CorpusReviewStatus.REJECTED; + rejectedExternalId = (item as RejectedCorpusItemPayload).externalId; + if ((item as RejectedCorpusItemPayload).reason) { + rejectionReasons = JSON.stringify( + (item as RejectedCorpusItemPayload).reason.split(','), + ); + } + } + + return { + object_version: 'new', + approved_corpus_item_external_id: isApproved ? approvedExternalId : '', + rejected_corpus_item_external_id: isApproved ? '' : rejectedExternalId, + prospect_id: item.prospectId ?? '', + url: item.url ?? '', + loaded_from: '', // TODO: Property 'source' does not exist on type 'ApprovedCorpusItemPayload | RejectedCorpusItemPayload' + corpus_review_status: corpusReviewStatus, + rejection_reasons_json: rejectionReasons, + action_screen: (item as any).action_screen ?? '', + title: item.title ?? '', + excerpt: isApproved + ? (item as ApprovedCorpusItemPayload).excerpt ?? '' + : '', + image_url: isApproved + ? (item as ApprovedCorpusItemPayload).imageUrl ?? '' + : '', + language: item.language ?? '', + topic: item.topic ?? '', + is_collection: isApproved + ? !!(item as ApprovedCorpusItemPayload).isCollection + : false, + is_syndicated: isApproved + ? !!(item as ApprovedCorpusItemPayload).isSyndicated + : false, + is_time_sensitive: isApproved + ? !!(item as ApprovedCorpusItemPayload).isTimeSensitive + : false, + created_at: item.createdAt + ? getUnixTimestamp(item.createdAt).toString() + : '', + created_by: item.createdBy ?? '', + updated_at: + isApproved && (item as ApprovedCorpusItemPayload).updatedAt + ? getUnixTimestamp( + (item as ApprovedCorpusItemPayload).updatedAt, + ).toString() + : '', + updated_by: isApproved + ? (item as ApprovedCorpusItemPayload).updatedBy ?? '' + : '', + authors_json: + isApproved && (item as ApprovedCorpusItemPayload).authors + ? JSON.stringify( + (item as ApprovedCorpusItemPayload).authors.map((a) => a.name), + ) + : '', + publisher: isApproved + ? (item as ApprovedCorpusItemPayload).publisher ?? '' + : '', + experimental_json: '', + }; + } +} From fcce75dffb36938529f62e9081b60bb7365f72c5 Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Tue, 18 Feb 2025 18:13:19 -0800 Subject: [PATCH 4/5] Add Python to build image --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a4558a45..a0fb76b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ ENV PNPM_HOME=/usr/local/bin RUN pnpm add -g turbo@2.1.0 #---------------------------------------- -# Docker build step that prunes down to +# Docker build step that prunes down to # the active project. #---------------------------------------- FROM base AS setup @@ -44,9 +44,9 @@ RUN apk update # Set working directory WORKDIR /app COPY . . -# Prune the structure to an optimized folder structure with just the `scopes` app dependencies. +# Prune the structure to an optimized folder structure with just the `scopes` app dependencies. RUN turbo prune --scope=$SCOPE --docker - + #---------------------------------------- # Docker build step that: # 1. Installs all the dependencies @@ -65,6 +65,8 @@ ARG SENTRY_PROJECT # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. RUN apk add --no-cache libc6-compat RUN apk update +# Install Python for @mozilla/glean code generation +RUN apk add --no-cache python3 WORKDIR /app # First install the dependencies (as they change less often) From fa0251e5c8b936af8033112dd07b6f12454b6cbb Mon Sep 17 00:00:00 2001 From: Mathijs Miermans Date: Wed, 19 Feb 2025 13:02:04 -0800 Subject: [PATCH 5/5] Change to json log message --- .../src/events/glean/ReviewedItemGleanHandler.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts index 1a6d015c..c84b7da3 100644 --- a/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts +++ b/servers/curated-corpus-api/src/events/glean/ReviewedItemGleanHandler.ts @@ -16,8 +16,9 @@ import { CorpusReviewStatus } from '../snowplow/schema'; // Create a custom stream that forwards messages to serverLogger. const customStream = { write: (message: string) => { - // Forward the log message to the existing serverLogger. - serverLogger.info(message); + // Parse the incoming message to send a jsonPayload to Cloud Logging. + const parsedMessage = JSON.parse(message); + serverLogger.info(parsedMessage); }, };