[add] URL crawler API & Lark Image component (#69)

Author: TechQuery

.env

@@ -1,5 +1,8 @@
 NEXT_PUBLIC_SITE_NAME = idea2app
 NEXT_PUBLIC_SITE_SUMMARY = 全行业信息化转型专家
+NEXT_PUBLIC_LOGO = https://github.com/idea2app.png
+
+NEXT_PUBLIC_CACHE_HOST = https://cache.idea2.app
 
 NEXT_PUBLIC_SENTRY_DSN = https://03e5d951172f411a04c1bab44022e22b@o4506471366852608.ingest.sentry.io/4506484563705856
 SENTRY_ORG = idea2app

README.md

@@ -68,6 +68,7 @@ You can check out [the Next.js GitHub repository][27] - your feedback and contri
 
 |           name           |     file     |       description       |
 | :----------------------: | :----------: | :---------------------: |
+|     `CRAWLER_TOKEN`      | `.env.local` | Web hooks authorization |
 |   `SENTRY_AUTH_TOKEN`    | `.env.local` | [Official document][28] |
 |       `SENTRY_ORG`       |    `.env`    | [Official document][29] |
 |     `SENTRY_PROJECT`     |    `.env`    | [Official document][29] |

components/Base/LarkImage.tsx

@@ -0,0 +1,35 @@
+import { TableCellValue } from 'mobx-lark';
+import { FC } from 'react';
+import { Image, ImageProps } from 'react-bootstrap';
+
+import { DefaultImage, fileURLOf } from '../../pages/api/Lark/file/[id]';
+
+export interface LarkImageProps extends Omit<ImageProps, 'src'> {
+  src?: TableCellValue;
+}
+
+export const LarkImage: FC<LarkImageProps> = ({
+  src = DefaultImage,
+  alt,
+  ...props
+}) => (
+  <Image
+    fluid
+    loading="lazy"
+    {...props}
+    src={fileURLOf(src, true)}
+    alt={alt}
+    onError={({ currentTarget: image }) => {
+      const path = fileURLOf(src),
+        errorURL = decodeURI(image.src);
+
+      if (!path) return;
+
+      if (errorURL.endsWith(path)) {
+        if (!alt) image.src = DefaultImage;
+      } else if (!errorURL.endsWith(DefaultImage)) {
+        image.src = path;
+      }
+    }}
+  />
+);

components/Client/Partner.tsx

@@ -1,8 +1,7 @@
 import { FC } from 'react';
-import { Image } from 'react-bootstrap';
 
 import { Client } from '../../models/Client';
-import { fileURLOf } from '../../pages/api/Lark/file/[id]';
+import { LarkImage } from '../Base/LarkImage';
 
 export interface PartnerProps extends Client {
   className?: string;
@@ -18,18 +17,18 @@ export const Partner: FC<PartnerProps> = ({
   <div
     className={`d-flex flex-column align-items-center justify-content-center gap-3 position-relative ${className}`}
   >
-    <Image
+    <LarkImage
       fluid
       className="object-fit-contain"
       style={{ height: '5rem' }}
       loading="lazy"
-      src={fileURLOf(image)}
+      src={image}
     />
     <h3>
       <a
         className="stretched-link"
         target="_blank"
-        href={address + ''}
+        href={address?.toString()}
         rel="noreferrer"
       >
         {name + ''}

package.json

@@ -12,6 +12,7 @@
     "koajax": "^3.1.1",
     "less": "^4.2.1",
     "less-loader": "^12.2.0",
+    "mime": "^4.0.4",
     "mobx": "^6.13.5",
     "mobx-github": "^0.3.5",
     "mobx-i18n": "^0.6.0",

pages/api/Lark/file/[id].ts

@@ -1,34 +1,50 @@
 import { fileTypeFromBuffer } from 'file-type';
+import MIME from 'mime';
 import { TableCellMedia, TableCellValue } from 'mobx-lark';
+import { parse } from 'path';
 
 import { safeAPI } from '../../core';
 import { lark } from '../core';
 
-export const fileURLOf = (field: TableCellValue) =>
-  field instanceof Array
-    ? field[0]
-      ? `/api/Lark/file/${(field[0] as TableCellMedia).file_token}`
-      : field + ''
-    : field + '';
+export const DefaultImage = process.env.NEXT_PUBLIC_LOGO!;
 
-export default safeAPI(async (req, res) => {
-  switch (req.method) {
+export function fileURLOf(field: TableCellValue, cache = false) {
+  if (!(field instanceof Array) || !field[0]) return field + '';
+
+  const { file_token, type } = field[0] as TableCellMedia;
+
+  let URI = `/api/Lark/file/${file_token}`;
+
+  if (cache) URI += '.' + MIME.getExtension(type);
+
+  return URI;
+}
+
+export const CACHE_HOST = process.env.NEXT_PUBLIC_CACHE_HOST!;
+
+export default safeAPI(async ({ method, url, query, headers }, res) => {
+  const { ext } = parse(url!);
+
+  if (ext)
+    return void res.redirect(
+      new URL(new URL(url!, `http://${headers.host}`).pathname, CACHE_HOST) +
+        '',
+    );
+  switch (method) {
+    case 'HEAD':
     case 'GET': {
-      const { id } = req.query;
+      const { id } = query;
 
       await lark.getAccessToken();
 
       const file = await lark.downloadFile(id as string);
 
-      const buffer = Buffer.alloc(file.byteLength),
-        view = new Uint8Array(file);
-
-      for (let i = 0; i < buffer.length; i++) buffer[i] = view[i];
-
-      const { mime } = (await fileTypeFromBuffer(buffer)) || {};
+      const { mime } = (await fileTypeFromBuffer(file)) || {};
 
       res.setHeader('Content-Type', mime as string);
-      res.send(buffer);
+
+      return void (method === 'GET' ? res.send(Buffer.from(file)) : res.end());
     }
   }
+  res.status(405).end();
 });

pages/api/crawler/task.ts

@@ -0,0 +1,32 @@
+import { githubClient } from 'mobx-github';
+
+import { safeAPI } from '../core';
+
+export interface CrawlerTask {
+  URI: string;
+  title?: string;
+}
+
+const CRAWLER_TOKEN = process.env.CRAWLER_TOKEN;
+
+export default safeAPI(async ({ method, headers, body }, response) => {
+  if (
+    !CRAWLER_TOKEN ||
+    CRAWLER_TOKEN !== headers.authorization?.split(/\s+/)[1]
+  )
+    return void response.status(401).end();
+
+  if (method !== 'POST') return void response.status(405).end();
+
+  const { URI, title = URI } = body as CrawlerTask;
+
+  const { status, body: data } = await githubClient.post(
+    'repos/idea2app/OWS-cache/issues',
+    {
+      title,
+      body: `### URL\n\n${URI}`,
+      labels: ['crawler'],
+    },
+  );
+  response.status(status).send(data);
+});

pages/project/[id].tsx

@@ -2,16 +2,16 @@ import { GitRepository } from 'mobx-github';
 import { observer } from 'mobx-react';
 import { compose, errorLogger, translator } from 'next-ssr-middleware';
 import { FC } from 'react';
-import { Col, Container, Image, Row } from 'react-bootstrap';
+import { Col, Container, Row } from 'react-bootstrap';
 
+import { LarkImage } from '../../components/Base/LarkImage';
 import { GitCard } from '../../components/Git/Card';
 import { PageHead } from '../../components/PageHead';
 import { ProjectCard } from '../../components/Project/Card';
 import { Project, ProjectModel } from '../../models/Project';
 import { GitRepositoryModel } from '../../models/Repository';
 import { i18n, t } from '../../models/Translation';
 import { solidCache } from '../api/core';
-import { fileURLOf } from '../api/Lark/file/[id]';
 
 interface ProjectDetailPageProps {
   project: Project;
@@ -58,7 +58,7 @@ const ProjectDetailPage: FC<ProjectDetailPageProps> = observer(
           className="text-decoration-none"
           href={project.link || '#'}
         >
-          <Image fluid src={fileURLOf(project.image)} alt={project.name + ''} />
+          <LarkImage fluid src={project.image} alt={project.name + ''} />
         </Col>
         <Col xs={12} sm={4}>
           <ProjectCard {...project} />