Merge pull request #232 from datakind/develop

release 0.3.6

Author: vishpillai123

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 0.3.6 (2025-06)
+- Fixed bug in features table (PR 229)
+- Fixed bug in 12 credit features (PR 230)
+
 ## 0.3.5 (2025-06)
 - Added support scores to to features (PR 222)
 - Limit boolean features to courses and subjects (PR 223)

pipelines/pdp/institution_id/00-data-assessment-TEMPLATE.py

@@ -25,7 +25,7 @@
 
 # install dependencies, of which most/all should come through our 1st-party SST package
 
-# %pip install "student-success-tool == 0.3.5"
+# %pip install "student-success-tool == 0.3.6"
 
 # COMMAND ----------
 

pipelines/pdp/institution_id/01-preprocess-data-TEMPLATE.py

@@ -24,7 +24,7 @@
 
 # install dependencies, of which most/all should come through our 1st-party SST package
 
-# %pip install "student-success-tool == 0.3.5"
+# %pip install "student-success-tool == 0.3.6"
 
 # COMMAND ----------
 

pipelines/pdp/institution_id/02-train-model-TEMPLATE.py

@@ -28,7 +28,7 @@
 # we need to manually install a certain version of pandas and scikit-learn in order
 # for our models to load and run properly.
 
-# %pip install "student-success-tool==0.3.5"
+# %pip install "student-success-tool==0.3.6"
 # %pip install "pandas==1.5.3"
 # %pip install "scikit-learn==1.3.0"
 

pipelines/pdp/institution_id/03-make-predictions-TEMPLATE.py

@@ -26,7 +26,7 @@
 # we need to manually install a certain version of pandas and scikit-learn in order
 # for our models to load and run properly.
 
-# %pip install "student-success-tool==0.3.5"
+# %pip install "student-success-tool==0.3.6"
 # %pip install "pandas==1.5.3"
 # %pip install "scikit-learn==1.3.0"
 

pipelines/pdp/institution_id/04-register-model-create-card-TEMPLATE.py

@@ -21,7 +21,7 @@
 # we need to manually install a certain version of pandas and scikit-learn in order
 # for our models to load and run properly.
 
-# %pip install "student-success-tool==0.3.5"
+# %pip install "student-success-tool==0.3.6"
 # %pip install "pandas==1.5.3"
 # %pip install "scikit-learn==1.3.0"
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "student-success-tool"
-version = "0.3.5"
+version = "0.3.6"
 description = "School-agnostic lib for implementing Student Success Tool workflows."
 readme = "README.md"
 requires-python = ">=3.10,<3.13"

src/student_success_tool/assets/pdp/features_table.toml

@@ -277,12 +277,11 @@ term_is_pre_cohort_cumsum = { name = "number of terms so far that occurred befor
 term_is_while_student_enrolled_at_other_inst = { name = "term occurred while student is enrolled at another institution", desc = "This is another binary feature (True/False) indicating whether a given (usually the checkpoint) academic term overlapped with a period when the student was concurrently enrolled at another institution. A value of \"True\" suggests simultaneous enrollment elsewhere in a given (usually the checkpoint) term, while a value of \"False\" means the student was not enrolled at another institution in a given (usually the checkpoint) term." }
 term_is_while_student_enrolled_at_other_inst_cumsum = { name = "number of terms so far that occurred while student was enrolled at another institution", desc = "This feature represents the cumulative number of academic terms in which a student was concurrently enrolled at another institution up until the given (usually the checkpoint) term. A higher value for this feature indicates that a student has taken several courses while enrolled elsewhere over their academic journey so far, while a lower value suggests limited or no overlap with enrollment at other institutions over a student's academic journey so far." }
 year_of_enrollment_at_cohort_inst = { name = "year of enrollment at cohort institution", desc = "The current year the student is enrolled in at the cohort insitution. Ex. year 1, year 2, year 3, etc." }
-'took_course_course_type_subject_area_(\d+)$' = { name = "took course in subject area '{}' this term", desc = "Whether the student has taken a course in a specified subject area in a given (usually the checkpoint) term. A 1 would indicate the student did take a course in this subject area and a 0 would mean the student did not." }
 term_program_of_study = { name = "student's program of study this term", desc = "Student’s term program of study this term based on major CIP code reported by the cohort institution." }
 term_program_of_study_area = { name = "student's program of study area this term", desc = "Student’s term program of study area this term based on major CIP code reported by the cohort institution." }
-'^took_course_course_id_([a-z]+)_(\d+)$' = { name = "course with id '{}{}' taken this term", desc = "This feature represents if a student has taken in a specific course in a given (usually the checkpoint) term, based on the course ID. A value of 1 for this feature would mean that a student took the specified course in a given (usually the checkpoint) term, while a value of 0 for this feature would mean that the student did not take the specified course in a given (usually the checkpoint) term." }
-'^took_course_course_id_([a-z]+)_(\d+)_cummax$' = { name = "course with id '{} {}' taken so far", desc = "This feature represents if a student has ever taken a specific course up until (usually the checkpoint) term, based on the course ID." }
-'^took_course_course_id_([a-z]+)_(\d+)_cummax_in_12_creds$' = { name = "course with id '{} {}' taken within student’s first 12 credits", desc = "This feature represents if a student has taken in a specific course in their first 12 credits, based on the course ID. A value of 1 for this feature would mean that a student took the specified course in their first 12 credits, while a value of 0 for this feature would mean that the student did not take the specified course in their first 12 credits." }
-'^took_course_course_subject_area_(\d+)$' = { name = "courses taken in subject area '{}' this term", desc = "Indicator whether student took a course in a specified subject area in a given (usually the checkpoint) term." }
-'^took_course_course_subject_area_(\d+)_cummax$' = { name = "course in subject area '{}' taken so far", desc = "Whether the student has taken a course in a specified subject area at any point up to this term. A 1 would indicate the student did take a course in this subject area and a 0 would mean the student did not." }
-'^took_course_course_subject_area_(\d+)_cummax_in_12_creds$' = { name = "took course in subject area '{}' within student’s first 12 credits", desc = "Whether the student has taken a course in a specified subject area within their first 12 credits. A 1 would indicate the student did take a course in this subject area during their first 12 credits and a 0 would mean the student did not." }
+'^took_course_id_([a-z]+)_(\d+)$' = { name = "course with id '{}{}' taken this term", desc = "This feature represents if a student has taken in a specific course in a given (usually the checkpoint) term, based on the course ID. A value of 1 for this feature would mean that a student took the specified course in a given (usually the checkpoint) term, while a value of 0 for this feature would mean that the student did not take the specified course in a given (usually the checkpoint) term." }
+'^took_course_id_([a-z]+)_(\d+)_cummax$' = { name = "course with id '{} {}' taken so far", desc = "This feature represents if a student has ever taken a specific course up until (usually the checkpoint) term, based on the course ID. A 1 would indicate the student did take the specified course with this course id at some point up until the given term and a 0 would mean the student did not." }
+'^took_course_id_([a-z]+)_(\d+)_cummax_in_12_creds$' = { name = "course with id '{} {}' taken within student’s first 12 credits", desc = "This feature represents if a student has taken in a specific course in their first 12 credits, based on the course ID. A value of 1 for this feature would mean that a student took the specified course in their first 12 credits, while a value of 0 for this feature would mean that the student did not take the specified course in their first 12 credits." }
+'^took_course_subject_area_(\d+)$' = { name = "courses taken in subject area '{}' this term", desc = "Indicator whether student took a course in a specified subject area in a given (usually the checkpoint) term. A 1 would indicate the student did take a course in this subject area in the given term and a 0 would mean the student did not." }
+'^took_course_subject_area_(\d+)_cummax$' = { name = "course in subject area '{}' taken so far", desc = "Whether the student has taken a course in a specified subject area at any point up to this term. A 1 would indicate the student did take a course in this subject area at some point up until the given term and a 0 would mean the student did not." }
+'^took_course_subject_area_(\d+)_cummax_in_12_creds$' = { name = "took course in subject area '{}' within student’s first 12 credits", desc = "Whether the student has taken a course in a specified subject area within their first 12 credits. A 1 would indicate the student did take a course in this subject area during their first 12 credits and a 0 would mean the student did not." }

src/student_success_tool/preprocessing/pdp.py

@@ -250,7 +250,7 @@ def clean_up_labeled_dataset_cols_and_vals(
         num_credits_col: Name of the column containing cumulative earned credits.
     """
     num_credit_check = constants.DEFAULT_COURSE_CREDIT_CHECK
-    credit_pattern = re.compile(rf"in_{num_credit_check}_credits")
+    credit_pattern = re.compile(rf"in_{num_credit_check}_creds")
     # To prevent data leakage, students that have not reached the 12 credits and not taken the course
     # by the checkpoint term (which this data is limited to at the time of this function),
     # will have the applicable in_12_credits columns set to null.

tests/assets/test_features_table.py

@@ -1,4 +1,5 @@
 import pytest
+import re
 import os
 from student_success_tool.dataio.read import from_toml_file
 
@@ -31,3 +32,43 @@ def test_all_features_have_name_and_desc(feature_table_data):
         assert "desc" in entry and entry["desc"].strip(), (
             f"'desc' missing or empty in feature: {feature_id}"
         )
+
+
+# Add regex test cases to this as needed! This helps us make sure
+# that are regex patterns work *as intended* in our features table
+VALID_FEATURE_NAMES = [
+    "took_course_id_eng_101",
+    "took_course_id_eng_101_cummax",
+    "took_course_id_eng_101_cummax_in_12_creds",
+    "took_course_subject_area_51",
+    "took_course_subject_area_51_cummax",
+    "took_course_subject_area_51_cummax_in_12_creds",
+    "num_courses_course_subject_area_51",
+    "num_courses_course_subject_area_51_cumfrac",
+    "frac_courses_course_subject_area_51",
+    "num_courses_course_id_eng_101",
+    "num_courses_course_id_eng_101_cumfrac",
+    "frac_courses_course_id_eng_101",
+]
+
+
+@pytest.mark.parametrize("feature_name", VALID_FEATURE_NAMES)
+def test_feature_matches_some_regex_key(feature_name, feature_table_data):
+    """Check if each valid feature name matches at least one of the TOML regex keys."""
+
+    def is_likely_regex(key: str) -> bool:
+        # Matches if the key contains metacharacters indicating it's a regex
+        return bool(re.search(r"[\(\[\.\*\+\?\\]", key))
+
+    # Only consider keys with \d in them — implying dynamic regex patterns
+    regex_keys = [key for key in feature_table_data.keys() if is_likely_regex(key)]
+
+    # Compile the regex patterns
+    compiled_patterns = [re.compile(pattern) for pattern in regex_keys]
+
+    # Check if the feature_name matches ANY of them
+    matched = any(pat.fullmatch(feature_name) for pat in compiled_patterns)
+
+    assert matched, (
+        f"Feature '{feature_name}' did not match any regex pattern with \\d in the TOML"
+    )