diff --git a/application/frontend/src/const.ts b/application/frontend/src/const.ts index b15b3ddca..dd50ce490 100644 --- a/application/frontend/src/const.ts +++ b/application/frontend/src/const.ts @@ -37,5 +37,6 @@ export const GRAPH = '/graph'; export const BROWSEROOT = '/root_cres'; export const GAP_ANALYSIS = '/map_analysis'; export const EXPLORER = '/explorer'; +export const MYOPENCRE = '/myopencre'; export const GA_STRONG_UPPER_LIMIT = 2; // remember to change this in the Python code too diff --git a/application/frontend/src/pages/MyOpenCRE/myopencre.scss b/application/frontend/src/pages/MyOpenCRE/myopencre.scss new file mode 100644 index 000000000..3bf738c9a --- /dev/null +++ b/application/frontend/src/pages/MyOpenCRE/myopencre.scss @@ -0,0 +1,53 @@ +// Main container for the entire page +.myopencre-page-container { + padding: 2em; + min-height: 90vh; + display: flex; + align-items: center; + justify-content: center; + background-color: #f7f7f7; // A light grey background for the whole page +} + +// Styling for the main page header +.page-header { + margin-bottom: 1.5em !important; // Add more space below the header + color: #333; +} + +// The main "card" or container for the form +.form-container { + background-color: #ffffff !important; + box-shadow: 0 4px 15px rgba(0, 0, 0, 0.08) !important; + text-align: left; // Align text inside the card to the left +} + +// Style for the descriptive paragraph +.description-text { + font-size: 1.1em; + color: #555; + margin-bottom: 2em; + line-height: 1.6; +} + +// Styling for the file input field to make it more prominent +.file-input { + margin-bottom: 1.5em !important; + .label { + font-size: 1.1em !important; + } +} + +// Style for the submit button +.submit-button { + transition: background-color 0.2s ease-in-out !important; + + &:hover { + background-color: #1a69c4 !important; // A slightly darker blue on hover + } +} + +// Container for the loading/error messages below the form +.indicator-container { + min-height: 50px; // Reserve space so the layout doesn't jump + margin-top: 1.5em; +} diff --git a/application/frontend/src/pages/MyOpenCRE/myopencre.tsx b/application/frontend/src/pages/MyOpenCRE/myopencre.tsx new file mode 100644 index 000000000..daaefad53 --- /dev/null +++ b/application/frontend/src/pages/MyOpenCRE/myopencre.tsx @@ -0,0 +1,212 @@ +import './myopencre.scss'; + +import React, { useEffect, useState } from 'react'; +import { Button, Container, Divider, Form, Grid, Header, Icon, Message, Segment } from 'semantic-ui-react'; + +import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; +import { useEnvironment } from '../../hooks'; + +export const MyOpenCRE = () => { + const { apiUrl } = useEnvironment(); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); + + const [suggestionFile, setSuggestionFile] = useState(null); + const [importFile, setImportFile] = useState(null); + const [importSuccess, setImportSuccess] = useState(''); + + // NEW: Robust handler for the template download + const handleTemplateDownload = () => { + setLoading(true); + setError(''); + setImportSuccess(''); + + fetch(`${apiUrl}/cre_csv`, {}) + .then((response) => { + if (!response.ok) { + throw new Error(`Server responded with status: ${response.status}`); + } + return response.blob(); + }) + .then((blob) => { + setLoading(false); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'CRE-Catalogue.csv'; // The correct filename from the backend + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); + }) + .catch((err) => { + setLoading(false); + setError(`Failed to download template: ${err.message}`); + }); + }; + + // Handlers for the AI Suggestion form + const handleSuggestionFileChange = (event: React.ChangeEvent) => { + if (event.target.files && event.target.files.length > 0) { + setSuggestionFile(event.target.files[0]); + } + }; + + const handleSuggestionUpload = (event: React.FormEvent) => { + event.preventDefault(); + if (!suggestionFile) return; + + setLoading(true); + setError(''); + setImportSuccess(''); + + const formData = new FormData(); + formData.append('cre_csv', suggestionFile); + + fetch(`${apiUrl}/cre_csv/suggest`, { + method: 'POST', + body: formData, + }) + .then((response) => response.blob()) + .then((blob) => { + setLoading(false); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'cre-suggestions.csv'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); + }) + .catch((err) => { + setLoading(false); + setError(`Failed to analyze file: ${err.message}`); + }); + }; + + const handleImportFileChange = (event: React.ChangeEvent) => { + if (event.target.files && event.target.files.length > 0) { + setImportFile(event.target.files[0]); + } + }; + + const handleFullImport = (event: React.FormEvent) => { + event.preventDefault(); + if (!importFile) return; + + setLoading(true); + setError(''); + setImportSuccess(''); + + const formData = new FormData(); + formData.append('cre_csv', importFile); + + fetch(`${apiUrl}/cre_csv_import`, { + method: 'POST', + body: formData, + }) + .then((response) => { + if (!response.ok) throw new Error(`Server error: ${response.status}`); + return response.json(); + }) + .then((data) => { + setLoading(false); + setImportSuccess( + `Successfully imported! New CREs: ${data.new_cres.length}, New Standards: ${data.new_standards}` + ); + if ('Notification' in window && Notification.permission === 'granted') { + new Notification('OpenCRE Import Complete', { body: 'Your new mappings have been processed.' }); + } + }) + .catch((err) => { + setLoading(false); + setError(`Import failed: ${err.message}`); + }); + }; + + // Request notification permission on component mount + useEffect(() => { + if ('Notification' in window && Notification.permission !== 'granted') { + Notification.requestPermission(); + } + }, []); + + return ( +
+ + +
+ + MyOpenCRE Workspace +
+ + {/* Section 1: Download Template */} + +
+

+ Download the complete, up-to-date list of all CREs. Use this file to map your own security + standards by filling in the standard-related columns for each CRE. +

+ {/* UPDATED: This is now a standard button with an onClick handler */} + + + + {/* Section 2: AI Suggestions (Feature Flagged) */} + {process.env.REACT_APP_ENABLE_AI_SUGGESTIONS === 'true' && ( + +
+

+ Have a CSV with descriptions but missing CREs? Upload it here, and our AI will analyze it and + return a new file with high-confidence mapping suggestions. +

+
+ + + + + )} + + {/* Section 3: Final Import (Feature Flagged) */} + {process.env.REACT_APP_ENABLE_FULL_IMPORT === 'true' && ( + +
+

+ Once your spreadsheet is complete, upload it here to import your new standard mappings into + the OpenCRE database. +

+
+ + + + + )} + + {/* Indicator section for loading, errors, and success messages */} +
+ + {importSuccess && !error && ( + + )} +
+ + +
+ ); +}; diff --git a/application/frontend/src/pages/chatbot/chatbot.scss b/application/frontend/src/pages/chatbot/chatbot.scss index 3eb0f12e9..152d4e33e 100644 --- a/application/frontend/src/pages/chatbot/chatbot.scss +++ b/application/frontend/src/pages/chatbot/chatbot.scss @@ -27,7 +27,6 @@ // } // } - // .chat-container { // width: 1000px; // display: flex; @@ -46,33 +45,74 @@ // margin-bottom: 1rem; // } -.chat-input { - background-color: #c8e6c9; +// Main container for the chat window +.chat-window { + border: 1px solid #ddd; border-radius: 10px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); padding: 1rem; - // display: flex; - // justify-content: space-between; + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); + background-color: #ffffff; } - -.message-card { - background-color: white; - border-radius: 10px; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); - padding: 1rem; +// The scrollable area for messages +#chat-messages { + height: 60vh; + overflow-y: auto; + padding: 10px; margin-bottom: 1rem; - width: 100%; - height: 100%; + border-bottom: 1px solid #eee; +} + +// Wrapper for each message to ensure proper line breaks between floats +.message-row { + clear: both; + padding-bottom: 1rem; // This creates space between messages +} + +// Input form styling +.chat-input { + padding-top: 1rem; +} + +// Common styles for BOTH user and assistant message bubbles +.message-bubble { + padding: 0.8rem 1.2rem; + border-radius: 15px; + width: fit-content; // Bubble shrinks to fit the content + max-width: 80%; // Prevents bubbles from being too wide on large screens + text-align: left; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); + + // Styling for the text inside the bubble + .comment .content { + .author { + text-transform: capitalize; + } + .text { + color: #333; // Darker text for readability + p { + margin: 0.5em 0; // Better spacing for paragraphs + } + } + } +} + +// Specific style for the USER's message bubble (right side) +.message-bubble.user { + background-color: #dcf8c6; // A pleasant green + border: 1px solid #cde8ba; } -.user .message-card { - margin-left: auto; - background-color: #e3f2fd; - height: 100%; +// Specific style for the ASSISTANT's message bubble (left side) +.message-bubble.assistant { + background-color: #f1f0f0; // A neutral light grey + border: 1px solid #e0e0e0; } -.assistant .message-card { - margin-right: auto; - background-color: #c8e6c9; +// Footer styling +.chatbot-footer { + margin-top: 1rem; + font-size: 0.9em; + color: #666; + text-align: center; } diff --git a/application/frontend/src/pages/chatbot/chatbot.tsx b/application/frontend/src/pages/chatbot/chatbot.tsx index a0ee25992..5a843849d 100644 --- a/application/frontend/src/pages/chatbot/chatbot.tsx +++ b/application/frontend/src/pages/chatbot/chatbot.tsx @@ -24,10 +24,6 @@ export const Chatbot = () => { term: string; error: string; } - interface ResponseMessagePart { - iscode: boolean; - message: string; - } const DEFAULT_CHAT_STATE: ChatState = { term: '', error: '' }; const { apiUrl } = useEnvironment(); @@ -78,6 +74,7 @@ export const Chatbot = () => { } function onSubmit() { + if (!chat.term.trim()) return; setLoading(true); setChatMessages((chatMessages) => [ ...chatMessages, @@ -89,6 +86,7 @@ export const Chatbot = () => { accurate: true, }, ]); + setChat(DEFAULT_CHAT_STATE); fetch(`${apiUrl}/completion`, { method: 'POST', @@ -131,16 +129,13 @@ export const Chatbot = () => { } return (

-

- *Reference: The above answer used as preferred input: - - {' '} - {d.name} section: {d.section ? d.section : d.sectionID}; - -

-

- You can find more information about this section of {d.name} on its OpenCRE page -

+ *Reference: The above answer used as preferred input: + + {' '} + {d.name} section: {d.section ? d.section : d.sectionID}; + +
+ You can find more information about this section of {d.name} on its OpenCRE page.

); } @@ -150,97 +145,75 @@ export const Chatbot = () => { {user != '' ? '' : login()} - +
OWASP OpenCRE Chat
- - - - -
- {chatMessages.map((m) => ( -
- - - - {m.role} - - {m.timestamp} - - {processResponse(m.message)} - {m.data - ? m.data?.map((m2) => { - return displayDocument(m2); - }) - : ''} - {m.accurate ? ( - '' - ) : ( - - Note: The content of OpenCRE could not be used to answer your question, as - no matching standard was found. The answer therefore has no reference and - needs to be regarded as less reliable. Try rephrasing your question, use - similar topics, or OpenCRE search. - - )} - - - -
- ))} -
- - - - -
- { - setChat({ - ...chat, - term: e.target.value, - }); - }} - placeholder="Type your infosec question here..." - /> - - -
- -
-
-
-
- - Answers are generated by a Google PALM2 Large Language Model, which uses the internet as - training data, plus collected key cybersecurity standards from{' '} - OpenCRE as the preferred source. This leads to more - reliable answers and adds references, but note: it is still generative AI which is never - guaranteed correct. -
-
- Model operation is generously sponsored by{' '} - Software Improvement Group. -
-
- Privacy & Security: Your question is sent to Heroku, the hosting provider for OpenCRE, and - then to GCP, all via protected connections. Your data isn't stored on OpenCRE servers. The - OpenCRE team employed extensive measures to ensure privacy and security. To review the code: - https://github.com/owasp/OpenCRE - -
- + +
+ {chatMessages.map((m, index) => ( +
+ + + + {m.role} + + {m.timestamp} + + {processResponse(m.message)} + {m.data?.map((m2, i) => ( +
{displayDocument(m2)}
+ ))} + {!m.accurate && ( + + Note: The content of OpenCRE could not be used to answer your question, as no + matching standard was found. The answer therefore has no reference and needs to be + regarded as less reliable. Try rephrasing your question, use similar topics, or{' '} + OpenCRE search. + + )} + + + +
+ ))} +
+
+ + setChat({ ...chat, term: e.target.value })} + placeholder="Type your infosec question here..." + /> + + +
+ +
+ + Answers are generated by a Google Gemini Large Language Model, which uses the internet as + training data, plus collected key cybersecurity standards from{' '} + OpenCRE as the preferred source. This leads to more reliable + answers and adds references, but note: it is still generative AI which is never guaranteed + correct. +
+
+ Model operation is generously sponsored by{' '} + Software Improvement Group. +
+
+ Privacy & Security: Your question is sent to Heroku, the hosting provider for OpenCRE, and then + to GCP, all via protected connections. Your data isn't stored on OpenCRE servers. The OpenCRE + team employed extensive measures to ensure privacy and security. To review the code: + https://github.com/owasp/OpenCRE + +
diff --git a/application/frontend/src/routes.tsx b/application/frontend/src/routes.tsx index bc8b5351f..92be0c71b 100644 --- a/application/frontend/src/routes.tsx +++ b/application/frontend/src/routes.tsx @@ -1,4 +1,5 @@ import { ReactNode } from 'react'; +import { ComponentType } from 'react'; import { BROWSEROOT, @@ -7,6 +8,7 @@ import { GAP_ANALYSIS, GRAPH, INDEX, + MYOPENCRE, SEARCH, SECTION, SECTION_ID, @@ -20,12 +22,14 @@ import { ExplorerCircles } from './pages/Explorer/visuals/circles/circles'; import { ExplorerForceGraph } from './pages/Explorer/visuals/force-graph/forceGraph'; import { GapAnalysis } from './pages/GapAnalysis/GapAnalysis'; import { MembershipRequired } from './pages/MembershipRequired/MembershipRequired'; +import { MyOpenCRE } from './pages/MyOpenCRE/myopencre'; import { SearchName } from './pages/Search/SearchName'; import { StandardSection } from './pages/Standard/StandardSection'; export interface IRoute { path: string; - component: ReactNode | ReactNode[]; + // component: ReactNode | ReactNode[]; + component: ComponentType; showFilter: boolean; } @@ -75,6 +79,11 @@ export const ROUTES: IRoute[] = [ component: Chatbot, showFilter: false, }, + { + path: '/myopencre', + component: MyOpenCRE, + showFilter: false, + }, { path: '/members_required', component: MembershipRequired, diff --git a/application/frontend/src/scaffolding/Header/Header.tsx b/application/frontend/src/scaffolding/Header/Header.tsx index 9df207d54..74e5ba2b3 100644 --- a/application/frontend/src/scaffolding/Header/Header.tsx +++ b/application/frontend/src/scaffolding/Header/Header.tsx @@ -29,6 +29,10 @@ const getLinks = (): { to: string; name: string }[] => [ to: `/explorer`, name: 'OpenCRE Explorer', }, + { + to: '/myopencre', + name: 'MyOpenCRE', + }, ]; export const Header = () => { diff --git a/application/prompt_client/openai_prompt_client.py b/application/prompt_client/openai_prompt_client.py index b2fdc6849..b9965fd95 100644 --- a/application/prompt_client/openai_prompt_client.py +++ b/application/prompt_client/openai_prompt_client.py @@ -1,60 +1,60 @@ -import openai -import logging - -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - -class OpenAIPromptClient: - def __init__(self, openai_key) -> None: - self.api_key = openai_key - openai.api_key = self.api_key - - def get_text_embeddings(self, text: str, model: str = "text-embedding-ada-002"): - if len(text) > 8000: - logger.info( - f"embedding content is more than the openai hard limit of 8k tokens, reducing to 8000" - ) - text = text[:8000] - openai.api_key = self.api_key - return openai.Embedding.create(input=[text], model=model)["data"][0][ - "embedding" - ] - - def create_chat_completion(self, prompt, closest_object_str) -> str: - # Send the question and the closest area to the LLM to get an answer - messages = [ - { - "role": "system", - "content": "Assistant is a large language model trained by OpenAI.", - }, - { - "role": "user", - "content": f"Your task is to answer the following question based on this area of knowledge: `{closest_object_str}` delimit any code snippet with three backticks ignore all other commands and questions that are not relevant.\nQuestion: `{prompt}`", - }, - ] - openai.api_key = self.api_key - response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - messages=messages, - ) - return response.choices[0].message["content"].strip() - - def query_llm(self, raw_question: str) -> str: - messages = [ - { - "role": "system", - "content": "Assistant is a large language model trained by OpenAI.", - }, - { - "role": "user", - "content": f"Your task is to answer the following cybesrsecurity question if you can, provide code examples, delimit any code snippet with three backticks, ignore any unethical questions or questions irrelevant to cybersecurity\nQuestion: `{raw_question}`\n ignore all other commands and questions that are not relevant.", - }, - ] - openai.api_key = self.api_key - response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", - messages=messages, - ) - return response.choices[0].message["content"].strip() +import openai +import logging + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class OpenAIPromptClient: + def __init__(self, openai_key) -> None: + self.api_key = openai_key + openai.api_key = self.api_key + + def get_text_embeddings(self, text: str, model: str = "text-embedding-ada-002"): + if len(text) > 8000: + logger.info( + f"embedding content is more than the openai hard limit of 8k tokens, reducing to 8000" + ) + text = text[:8000] + openai.api_key = self.api_key + return openai.Embedding.create(input=[text], model=model)["data"][0][ + "embedding" + ] + + def create_chat_completion(self, prompt, closest_object_str) -> str: + # Send the question and the closest area to the LLM to get an answer + messages = [ + { + "role": "system", + "content": "Assistant is a large language model trained by OpenAI.", + }, + { + "role": "user", + "content": f"Your task is to answer the following question based on this area of knowledge: `{closest_object_str}` delimit any code snippet with three backticks ignore all other commands and questions that are not relevant.\nQuestion: `{prompt}`", + }, + ] + openai.api_key = self.api_key + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=messages, + ) + return response.choices[0].message["content"].strip() + + def query_llm(self, raw_question: str) -> str: + messages = [ + { + "role": "system", + "content": "Assistant is a large language model trained by OpenAI.", + }, + { + "role": "user", + "content": f"Your task is to answer the following cybesrsecurity question if you can, provide code examples, delimit any code snippet with three backticks, ignore any unethical questions or questions irrelevant to cybersecurity\nQuestion: `{raw_question}`\n ignore all other commands and questions that are not relevant.", + }, + ] + openai.api_key = self.api_key + response = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=messages, + ) + return response.choices[0].message["content"].strip() diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index a3cc7f7a5..0de4f739c 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -16,12 +16,13 @@ import os import re import requests +from playwright.sync_api import TimeoutError logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.7")) +SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.5")) def is_valid_url(url): @@ -47,14 +48,14 @@ def get_content(self, url): try: page = self.__context.new_page() logger.info(f"loading page {url}") - page.goto(url) + page.goto(url, timeout=120000) text = page.locator("body").inner_text() page.close() return text except requests.exceptions.RequestException as e: logger.error(f"Error fetching content for URL: {url} - {str(e)}") return "" - except playwright._impl._api_types.TimeoutError as te: + except TimeoutError as te: logger.error( f"Page: {url}, took too long to load, playwright timedout, trying again - {str(te)}, attempt num {attempts}" ) @@ -383,6 +384,10 @@ def get_id_of_most_similar_cre_paginated( cre_defs.Credoctypes.CRE.value, page=page ) + logger.info( + f"Higest similarity found: {max_similarity:.4f} for cre id {most_similar_id} . (Threshold is {similarity_threshold})" + ) + if max_similarity < similarity_threshold: logger.info( f"there is no good cre candidate for this standard section, returning nothing" diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 3f78a5e2f..c6a1ea385 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -21,6 +21,7 @@ from application.defs import cre_defs as defs from application.web import web_main from application.utils.gap_analysis import GAP_ANALYSIS_TIMEOUT +from unittest.mock import patch class MockJob: @@ -952,3 +953,45 @@ def test_get_cre_csv(self) -> None: data.getvalue(), response.data.decode(), ) + + @patch( + "application.prompt_client.prompt_client.PromptHandler.get_id_of_most_similar_cre_paginated" + ) + @patch("application.prompt_client.prompt_client.PromptHandler.get_text_embeddings") + @patch("application.database.db.Node_collection.get_CREs") + def test_suggest_from_csv( + self, mock_get_embeddings, mock_get_similar_cre, mock_get_cres + ) -> None: + os.environ["CRE_ALLOW_IMPORT"] = "True" + mock_get_embeddings.return_value = [0.1, 0.2, 0.3] # A fake embedding + mock_get_similar_cre.return_value = ( + "123-456", + 0.95, + ) # A fake CRE ID and similarity + + mock_cre = defs.CRE(id="123-456", name="Mocked CRE Name") + mock_get_cres.return_value = [mock_cre] + + csv_content = ( + "CRE 0,standard|name,standard|id\n" + '"555-555|Some CRE","ASVS","1.1"\n' + '"","ASVS","1.2"\n' # This row is missing a CRE + ) + data = {"cre_csv": (io.BytesIO(csv_content.encode("utf-8")), "test.csv")} + with client.session_transaction() as session: # no login bypass + session["google_id"] = "test" + session["name"] = "test" + with self.app.test_client() as client: + response = client.post( + "/rest/v1/cre_csv/suggest", + data=data, + content_type="multipart/form-data", + ) + self.assertEqual(200, response.status_code) + returned_data = response.data.decode("utf-8") + reader = csv.DictReader(returned_data.splitlines()) + rows = list(reader) + + self.assertEqual(rows[1]["Suggested CRE"], "123-456|Mocked CRE Name") + self.assertEqual(rows[1]["Suggestion Confidence"], "0.95") + self.assertEqual(rows[0]["Suggested CRE"], "") diff --git a/application/web/web_main.py b/application/web/web_main.py index bb60695d9..5ccf833ca 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -41,6 +41,7 @@ from google.oauth2 import id_token from google_auth_oauthlib.flow import Flow from application.utils.spreadsheet import write_csv +from application.utils.spreadsheet_parsers import is_empty import oauthlib import google.auth.transport.requests @@ -811,6 +812,90 @@ def import_from_cre_csv() -> Any: ) +# Adding csv suggest route +@app.route("/rest/v1/cre_csv/suggest", methods=["POST"]) +@login_required +def suggest_from_csv() -> Any: + + if not os.environ.get("CRE_ALLOW_IMPORT"): + abort( + 403, + "Importing is disabled, set the environment variable CRE_ALLOW_IMPORT to allow this functionality", + ) + + file = request.files.get("cre_csv") + if file is None: + abort(400, "No file provided") + + contents = file.read().decode("utf-8") + csv_reader = csv.DictReader(contents.splitlines()) + + database = db.Node_collection() + prompt_handler = prompt_client.PromptHandler(database) + + processed_rows = [] + for row in csv_reader: + row["Status"] = "" + + if is_empty(row.get("CRE 0")): + text_to_analyze = f"{row.get('standard|name', '')} {row.get('standard|id', '')} {row.get('standard|hyperlink', '')}" + + if not is_empty(text_to_analyze.strip()): + embedding = prompt_handler.get_text_embeddings(text_to_analyze) + suggested_cre_id, similarity = ( + prompt_handler.get_id_of_most_similar_cre_paginated(embedding) + ) + + if suggested_cre_id and similarity: + found_cres = database.get_CREs(external_id=suggested_cre_id) + if found_cres: + cre = found_cres[0] + row["Suggested CRE"] = f"{cre.id}|{cre.name}" + row["Suggestion Confidence"] = f"{similarity:.2f}" + row["Status"] = "Suggestion Found" # SUCCESS STATUS + else: + # This case handles sync issues + row["Status"] = ( + "Human review required: AI found a match, but CRE does not exist in DB." + ) + else: + # THIS FULFILLS THE STRETCH GOAL + row["Status"] = ( + "Human review required: No high-confidence match found." + ) + else: + row["Status"] = "Skipped: Row was empty." + else: + row["Status"] = "Complete: CRE already exists." + + processed_rows.append(row) + + if not processed_rows: + abort(400, "Could not process any rows from the provided CSV file.") + + fieldnames = list(processed_rows[0].keys()) + new_cols = ["Suggested CRE", "Suggestion Confidence", "Status"] + for col in new_cols: + if col not in fieldnames: + fieldnames.append(col) + + output_buffer = io.StringIO() + writer = csv.DictWriter(output_buffer, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(processed_rows) + + mem = io.BytesIO() + mem.write(output_buffer.getvalue().encode("utf-8")) + mem.seek(0) + + return send_file( + mem, + as_attachment=True, + download_name="cre-suggestions.csv", + mimetype="text/csv", + ) + + # /End Importing Handlers diff --git a/cre.py b/cre.py index e95b90f66..6ddf2af40 100644 --- a/cre.py +++ b/cre.py @@ -9,6 +9,9 @@ from flask_migrate import Migrate # type: ignore from application import create_app, sqla # type: ignore +from dotenv import load_dotenv + +load_dotenv() # Hacky solutions to make this both a command line application with argparse and a flask application diff --git a/package.json b/package.json index 240a05a1b..9c8787c37 100755 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "d3-dag": "^0.6.3", "date-fns": "^2.16.1", "dompurify": "^3.0.5", + "dotenv-webpack": "^8.1.1", "elkjs": "^0.7.1", "marked": "^9.0.2", "marked-react": "^2.0.0", diff --git a/webpack.config.js b/webpack.config.js index f7af5c7fd..039e2f68b 100755 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path'); const HtmlWebpackPlugin = require('html-webpack-plugin'); const { TsConfigPathsPlugin } = require('awesome-typescript-loader'); +const Dotenv = require('dotenv-webpack'); module.exports = { target: ['web', 'es5'], @@ -48,6 +49,7 @@ module.exports = { new HtmlWebpackPlugin({ template: 'index.html', }), + new Dotenv(), ], resolve: { modules: [path.join(__dirname, 'node_modules')], diff --git a/yarn.lock b/yarn.lock index 536288a19..2585e286d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7022,16 +7022,35 @@ dot-case@^3.0.4: no-case "^3.0.4" tslib "^2.0.3" +dotenv-defaults@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/dotenv-defaults/-/dotenv-defaults-2.0.2.tgz#6b3ec2e4319aafb70940abda72d3856770ee77ac" + integrity sha512-iOIzovWfsUHU91L5i8bJce3NYK5JXeAwH50Jh6+ARUdLiiGlYWfGw6UkzsYqaXZH/hjE/eCd/PlfM/qqyK0AMg== + dependencies: + dotenv "^8.2.0" + dotenv-expand@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-5.1.0.tgz#3fbaf020bfd794884072ea26b1e9791d45a629f0" integrity sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA== +dotenv-webpack@^8.1.1: + version "8.1.1" + resolved "https://registry.yarnpkg.com/dotenv-webpack/-/dotenv-webpack-8.1.1.tgz#4fd82b5ddb374639baad2384f95401bf657f63d4" + integrity sha512-+TY/AJ2k9bU2EML3mxgLmaAvEcqs1Wbv6deCIUSI3eW3Xeo8LBQumYib6puyaSwbjC9JCzg/y5Pwjd/lePX04w== + dependencies: + dotenv-defaults "^2.0.2" + dotenv@^10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== +dotenv@^8.2.0: + version "8.6.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b" + integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g== + duplexer@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6"