Development: Run PECV Bench in Artemis

supporting_scripts/course-scripts/hyperion-benchmark-workflow/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+.env
+venv
+.idea/
+pecv-bench/

supporting_scripts/course-scripts/hyperion-benchmark-workflow/README.md

@@ -0,0 +1,76 @@
+# Artemis Course Setup Scripts
+
+This project contains Python scripts that automate the PECV Bench Course setup and management of programming exercises in an Artemis instance.
+
+# Setup
+
+## Prerequisites
+
+
+- Python 3.13 (other versions might not work due to model incompatibility)
+- pip
+
+## 1. Create and activate a virtual environment in **hyperion-benchmark-workflow** folder
+It is recommended to use a virtual environment to manage dependencies in isolation from the global Python environment. This approach can prevent version conflicts and keep your system environment clean.
+
+1. Install `virtualenv` if it's not already installed:
+   - If you are using [brew](https://brew.sh/) on macOS, you can install `virtualenv` using:
+      ```shell
+      brew install virtualenv
+      ```
+   - Otherwise, you can use pip to install `virtualenv`:
+      ```shell
+      python3 -m pip install virtualenv
+      ```
+
+2. Create a virtual environment in **hyperion-benchmark-workflow** folder:
+    ```shell
+    python3 -m venv venv
+    ```
+
+3. Activate the virtual environment:
+   - On **macOS/Linux**:
+     ```shell
+     source venv/bin/activate
+     ```
+   - On **Windows**:
+     ```shell
+     venv\Scripts\activate
+     ```
+
+Once the virtual environment is activated, you will see the `(venv)` prefix in your terminal prompt. All dependencies will now be installed locally to this environment.
+
+## 2. Install the Required Packages
+```shell
+pip install -r requirements.txt
+```
+
+## 3. Configure the Environment
+1. Start your local Artemis instance.
+2. Configure the values in [hyperion-benchmark-workflow/config.ini](./config.ini) according to your local setup.
+
+
+**Note:**
+1. Ensure that the [hyperion-benchmark-workflow/config.ini](./config.ini) file is correctly configured before running any scripts.
+2. **Always test the scripts on a local setup before running them on a production or test server! ⚠️**
+
+# Usage 
+The script will automatically perform all the necessary steps (running from hyperion-benchmark-workflow):
+
+```shell
+python3 run_pecv_bench_in_artemis.py
+```
+
+1. Authenticate as admin. 
+2. Create course.
+3. Materialize variants
+4. Convert variants to zip files
+5. Import exercise from zip file
+6. Execute consistency check on each exercise
+7. Run reporting scripts to generate quantifiable metrics for analysis and save them in markdown file
+
+Analysis results are stored inside *hyperion-benchmark-workflow/pecv-bench/results/artemis-benchmark/* folder
+* report.md
+* summary.md/json/tex
+* variants_report_plots
+* variants_report.json

supporting_scripts/course-scripts/hyperion-benchmark-workflow/config.ini

@@ -0,0 +1,72 @@
+[Settings]
+# ================================
+# USER SETTINGS
+# ================================
+# These credentials are used for authentication.
+# Ensure that the values match those on the test server if necessary.
+admin_user = artemis_admin
+admin_password = artemis_admin
+
+# ================================
+# THREADING SETTINGS
+# ================================
+# Defines the number of concurrent threads for creating or deleting users.
+# Adjust based on server capacity to optimize performance.
+max_threads = 5
+
+# ================================
+# URL SETTINGS
+# ================================
+# Base URLs for the server and client applications.
+# Modify these values if deploying to a different environment.
+server_url = http://localhost:8080/api
+client_url = http://localhost:9000/api
+
+# Example URLs for a test server.
+# Uncomment and use these if running the script on a test environment.
+;server_url = https://artemis-test1.artemis.cit.tum.de/api
+;client_url = https://artemis-test1.artemis.cit.tum.de/api
+
+
+[CourseSettings]
+# ================================
+# COURSE CONFIGURATION
+# ================================
+
+# Regular expression to filter special characters in course names.
+# Can be left as is unless a different filtering rule is required.
+special_character_regex = r'[^a-zA-Z0-9_]'
+
+# Whether to create a new course automatically.
+# Set to 'True' to enable course creation, otherwise set to 'False'
+# and specify an existing course ID below.
+create_course = True
+
+# If create_course is set to 'False', this ID will be used to select an existing course.
+course_id = 1234
+
+# Name of the course if create_course is set to 'True'.
+# Modify this if a different name is desired when creating a course.
+course_name = PECV Bench Course
+
+# Defines whether the course is a local course.
+# Set to 'False' when creating a course on a test server,
+# as this affects user group configurations.
+is_local_course = True
+
+[PECVBenchSettings]
+# ================================
+# PECV-BENCH CONFIGURATION
+# ================================
+
+# Directory name for the PECV-bench repository.
+pecv_bench_folder = pecv-bench
+
+# URL of the PECV-bench Git repository.
+pecv_bench_repo = "https://github.com/ls1intum/PECV-bench.git"
+
+# course and exercise identifiers for variant creation
+course = ITP2425
+exercises = H01E01-Lectures,H02E02-Panic_at_Seal_Saloon,H05E01-Space_Seal_Farm
+
+reference = | pecv-reference | model=openai:o4-mini, reasoning_effort=medium | 3 | 254 | 148 | 25 | 0.632 | 0.910 | 0.746 | 0.676 | 0.565 | 32.958 | 0.0338 |

supporting_scripts/course-scripts/hyperion-benchmark-workflow/create_pecv_bench_course.py

@@ -0,0 +1,155 @@
+import requests
+import configparser
+import json
+import urllib3
+import re
+import time
+from logging_config import logging
+from requests import Session
+
+# Load configuration
+config = configparser.ConfigParser()
+config.read(['config.ini'])
+
+# Constants from config file
+SERVER_URL: str = config.get('Settings', 'server_url')
+IS_LOCAL_COURSE: bool = config.get('CourseSettings', 'is_local_course').lower() == 'true'  # Convert to boolean
+COURSE_NAME: str = config.get('CourseSettings', 'course_name')
+SPECIAL_CHARACTERS_REGEX: str = config.get('CourseSettings', 'special_character_regex')
+
+def parse_course_name_to_short_name() -> str:
+    """Parse course name to create a short name, removing special characters."""
+    short_name = COURSE_NAME.strip()
+    short_name = re.sub(SPECIAL_CHARACTERS_REGEX, '', short_name.replace(' ', ''))
+
+    if len(short_name) > 0 and not short_name[0].isalpha():
+        short_name = 'a' + short_name
+
+    return short_name
+
+def create_pecv_bench_course(session: Session) -> requests.Response:
+    """Create a course using the given session."""
+    url = f"{SERVER_URL}/core/admin/courses"
+    course_short_name = parse_course_name_to_short_name()
+
+    default_course = {
+        "id": None,
+        "title": str(COURSE_NAME),
+        "shortName": str(course_short_name),
+        "customizeGroupNames": False,
+        "studentGroupName": None,
+        "teachingAssistantGroupName": None,
+        "editorGroupName": None,
+        "instructorGroupName": None,
+        "courseInformationSharingMessagingCodeOfConduct": None,
+        "semester": None,
+        "testCourse": None,
+        "onlineCourse": False,
+        "complaintsEnabled": True,
+        "requestMoreFeedbackEnabled": True,
+        "maxPoints": None,
+        "accuracyOfScores": 1,
+        "defaultProgrammingLanguage": None,
+        "maxComplaints": 3,
+        "maxTeamComplaints": 3,
+        "maxComplaintTimeDays": 7,
+        "maxComplaintTextLimit": 2000,
+        "maxComplaintResponseTextLimit": 2000,
+        "maxRequestMoreFeedbackTimeDays": 7,
+        "registrationConfirmationMessage": None,
+        "unenrollmentEnabled": None,
+        "color": None,
+        "courseIcon": None,
+        "timeZone": None,
+        "courseInformationSharingConfiguration": "COMMUNICATION_AND_MESSAGING",
+        "enrollmentEnabled": False
+    }
+
+    if IS_LOCAL_COURSE:
+        default_course["customizeGroupNames"] = True
+        default_course["studentGroupName"] = "students"
+        default_course["teachingAssistantGroupName"] = "tutors"
+        default_course["editorGroupName"] = "editors"
+        default_course["instructorGroupName"] = "instructors"
+
+    fields = {
+        "course": ('blob.json', json.dumps(default_course), 'application/json')
+    }
+    body, content_type = urllib3.filepost.encode_multipart_formdata(fields)
+    headers = {
+        'Content-Type': content_type,
+    }
+
+    logging.info(f"Creating course {COURSE_NAME} with shortName {course_short_name}")
+
+    response: requests.Response = session.post(url, data=body, headers=headers)
+
+    if response.status_code == 400:
+        logging.error(f"Course with shortName {course_short_name} already exists")
+
+        course_is_deleted = delete_pecv_bench_course(session, course_short_name)
+
+        if course_is_deleted:
+            logging.info(f"Waiting 2 seconds for server cleanup before recreation")
+            time.sleep(2)
+
+            logging.info(f"RETRYING: Creating course {COURSE_NAME} with shortName {course_short_name}")
+            response: requests.Response = session.post(url, data=body, headers=headers)
+        else:
+            raise Exception(f"Failed to delete existing course {COURSE_NAME} after multiple attempts. Cannot proceed")
+
+    # final check for successful creation
+    if response.status_code == 201:
+        logging.info(f"Successfully created course '{COURSE_NAME}' (ShortName: {course_short_name})")
+        return response.json()
+    else:        
+        raise Exception(
+                f"Could not create course {COURSE_NAME}; Status code: {response.status_code}\n"
+                f"Double check whether the courseShortName {course_short_name} is valid (e.g. no special characters such as '-')!\n"
+                f"Response content: {response.text}")
+
+
+def delete_pecv_bench_course(session: Session, course_short_name: str, max_retries: int = 3) -> bool:
+    """Delete a course with retry logic using the given session and course ID.
+
+    Sometimes it takes multiple scripts reruns to successfully delete a course.
+    This approach fixes this.
+    """
+
+    logging.info(f"Attempting to delete course with shortName {course_short_name}")
+
+    courseResponse: requests.Response = session.get(f"{SERVER_URL}/core/courses")
+    courses = courseResponse.json()
+    course_id = None
+    for course in courses:
+        if course["shortName"] == course_short_name:
+            course_id = course["id"]
+            break
+    if course_id is None:
+        logging.error(f"Course with shortName {course_short_name} not found")
+        return False
+
+    delete_url = f"{SERVER_URL}/core/admin/courses/{course_id}"
+    logging.info(f"Deleting course with ID {course_id} at URL {delete_url}")
+
+    for attempt in range(1, max_retries +1):
+        logging.info(f"Deletion attempt {attempt}/{max_retries}")
+        try:
+            # If server takes longer than 60s, it raises a Timeout exception, triggering the retry logic.
+            logging.info(f"Sending DELETE request, it can take around 30 seconds to delete a course")
+            deleteCourseResponse: requests.Response = session.delete(delete_url, timeout = 60)
+            if deleteCourseResponse.status_code == 200:
+                logging.info(f"DELETED: on attempt {attempt} course with shortName {course_short_name} was deleted successfully ")
+                return True
+            else:
+                logging.error(f"Deletion attempt {attempt}/{max_retries} for {course_short_name} failed with status code {deleteCourseResponse.status_code}")
+        except Exception as e:
+            logging.exception(f"Exception during deletion attempt {attempt}/{max_retries} for {course_short_name}: {e}")
+
+        if attempt < max_retries:
+            wait_time = 2 * attempt
+            logging.info(f"Waiting {wait_time} seconds before retrying deletion...")
+            time.sleep(wait_time)
+    logging.error(f"Failed to delete course with shortName {course_short_name} after {max_retries} attempts.")
+    return False
+

supporting_scripts/course-scripts/hyperion-benchmark-workflow/logging_config.py

@@ -0,0 +1,4 @@
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')