fix example

Author: jtwinrelevanceai

examples/fail_example.py

@@ -40,7 +40,8 @@ def __init__(
 
     def transform(self, documents: List[Document]) -> List[Document]:
         try:
-            text = [document[self.text_field] for document in documents]
+            raise ValueError
+
         except:
             # pass
             raise UserFacingError(

tests/conftest.py

@@ -78,6 +78,21 @@ def partial_dataset(test_client: Client) -> Dataset:
     test_client.delete_dataset(dataset_id)
 
 
+@pytest.fixture(scope="class")
+def simple_partial_dataset(test_client: Client) -> Dataset:
+    salt = "".join(random.choices(string.ascii_lowercase, k=10))
+    dataset_id = f"_sample_dataset_{salt}"
+    dataset = test_client.Dataset(dataset_id, expire=True)
+    documents = mock_documents(1000)
+    fields = ["sample_1_label"]
+    for document in documents:
+        for field in random.sample(fields, k=random.randint(1, 3)):
+            document.pop(field)
+    dataset.insert_documents(documents)
+    yield dataset
+    test_client.delete_dataset(dataset_id)
+
+
 @pytest.fixture(scope="class")
 def partial_dataset_with_outputs(test_client: Client) -> Dataset:
     salt = "".join(random.choices(string.ascii_lowercase, k=10))
@@ -297,7 +312,7 @@ def test_user_facing_error_workflow_token(test_client: Client) -> str:
         job_id=job_id,
         dataset_id=dataset_id,
         authorizationToken=test_client.credentials.token,
-        text_field="sample_1_description_not_in_dataset",
+        text_field="sample_1_description",
     )
     config_string = json.dumps(config)
     config_bytes = config_string.encode()

tests/core/test_engine/test_stable_engine.py

@@ -62,3 +62,19 @@ def test_stable_engine_filters2(
             assert prev_health[input_field]["exists"] == post_health[output_field]["exists"]
 
         assert engine.success_ratio == 1
+
+    def test_stable_engine_filters3(
+        self, simple_partial_dataset: Dataset, test_partial_operator: Type[AbstractOperator]
+    ):
+        prev_health = simple_partial_dataset.health()
+        operator = test_partial_operator(["sample_1_label"])
+
+        engine = StableEngine(simple_partial_dataset, operator, select_fields=["sample_1_label"], refresh=False)
+        workflow = Workflow(name=_random_id(), engine=engine, job_id=_random_id())
+        workflow.run()
+
+        post_health = simple_partial_dataset.health()
+        for input_field, output_field in zip(operator.input_fields, operator.output_fields):
+            assert prev_health[input_field]["exists"] == post_health[output_field]["exists"]
+
+        assert engine.success_ratio == 1