Skip to content

Commit 68c20b3

Browse files
MarceloRobertMarceloRobert
authored
Feat: treeDetails lab filter (#1618)
* feat: add build labs return and filters - Adds build lab on summary and filters returns - Adds filter logic to build lab in backend * refactor: reuse OriginsCard as generic FiltersCard * feat: add treeDetails build lab cards * feat: add boot and test lab return and filter - Adds boot and test lab in summary and filters returns - Adds filter logic for boot and test lab * feat: add treeDetails boot and test lab cards Closes #1616 * docs: add notes for new filters Also fixes the integration tests notes
1 parent 1b64947 commit 68c20b3

29 files changed

+354
-74
lines changed

‎backend/kernelCI_app/helpers/filters.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,10 @@ def __init__(self, data: Dict, process_body=False) -> None:
353353
self.filter_boot_origin: set[str] = set()
354354
self.filter_test_origin: set[str] = set()
355355

356+
self.filter_build_lab: set[str] = set()
357+
self.filter_boot_lab: set[str] = set()
358+
self.filter_test_lab: set[str] = set()
359+
356360
self.filter_handlers: FilterHandlers = {
357361
"boot.status": self._handle_boot_status,
358362
"boot.duration": self._handle_boot_duration,
@@ -378,6 +382,9 @@ def __init__(self, data: Dict, process_body=False) -> None:
378382
"build.origin": self._handle_build_origin,
379383
"boot.origin": self._handle_boot_origin,
380384
"test.origin": self._handle_test_origin,
385+
"build.lab": self._handle_build_lab,
386+
"boot.lab": self._handle_boot_lab,
387+
"test.lab": self._handle_test_lab,
381388
}
382389

383390
self.filters: List[FilterParams.ParsedFilter] = []
@@ -486,6 +493,15 @@ def _handle_boot_origin(self, current_filter: ParsedFilter) -> None:
486493
def _handle_test_origin(self, current_filter: ParsedFilter) -> None:
487494
self.filter_test_origin.add(current_filter["value"])
488495

496+
def _handle_build_lab(self, current_filter: ParsedFilter) -> None:
497+
self.filter_build_lab.add(current_filter["value"])
498+
499+
def _handle_boot_lab(self, current_filter: ParsedFilter) -> None:
500+
self.filter_boot_lab.add(current_filter["value"])
501+
502+
def _handle_test_lab(self, current_filter: ParsedFilter) -> None:
503+
self.filter_test_lab.add(current_filter["value"])
504+
489505
def _process_filters(self):
490506
try:
491507
for current_filter in self.filters:
@@ -603,6 +619,7 @@ def is_build_filtered_out(
603619
issue_version: Optional[int],
604620
incident_test_id: Optional[str],
605621
build_origin: Optional[str] = None,
622+
build_lab: Optional[str] = None,
606623
) -> bool:
607624
return (
608625
(
@@ -637,6 +654,10 @@ def is_build_filtered_out(
637654
build_status=build_status,
638655
)
639656
)
657+
or (
658+
len(self.filter_build_lab) > 0
659+
and (build_lab not in self.filter_build_lab)
660+
)
640661
)
641662

642663
def is_record_filtered_out(
@@ -694,6 +715,7 @@ def is_boot_filtered_out(
694715
incident_test_id: Optional[str] = "incident_test_id",
695716
platform: Optional[str] = None,
696717
origin: Optional[str] = None,
718+
lab: Optional[str] = None,
697719
) -> bool:
698720
if (
699721
(self.filterBootPath != "" and (self.filterBootPath not in path))
@@ -729,6 +751,7 @@ def is_boot_filtered_out(
729751
len(self.filter_boot_origin) > 0
730752
and (origin not in self.filter_boot_origin)
731753
)
754+
or (len(self.filter_boot_lab) > 0 and (lab not in self.filter_boot_lab))
732755
):
733756
return True
734757

@@ -745,6 +768,7 @@ def is_test_filtered_out(
745768
incident_test_id: Optional[str] = "incident_test_id",
746769
platform: Optional[str] = None,
747770
origin: Optional[str] = None,
771+
lab: Optional[str] = None,
748772
) -> bool:
749773
if (
750774
(self.filterTestPath != "" and (self.filterTestPath not in path))
@@ -780,6 +804,7 @@ def is_test_filtered_out(
780804
len(self.filter_test_origin) > 0
781805
and (origin not in self.filter_test_origin)
782806
)
807+
or (len(self.filter_test_lab) > 0 and (lab not in self.filter_test_lab))
783808
):
784809
return True
785810

‎backend/kernelCI_app/helpers/hardwareDetails.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,7 @@ def generate_build_summary_typed() -> BuildSummary:
226226
configs={},
227227
issues=[],
228228
unknown_issues=0,
229+
labs={},
229230
)
230231

231232

@@ -239,6 +240,7 @@ def generate_test_summary_typed() -> TestSummary:
239240
unknown_issues=0,
240241
fail_reasons={},
241242
failed_platforms=set(),
243+
labs={},
242244
)
243245

244246

@@ -298,7 +300,11 @@ def handle_test_history(
298300
environment_misc=EnvironmentMisc(platform=record["test_platform"]),
299301
tree_name=record["build__checkout__tree_name"],
300302
git_repository_branch=record["build__checkout__git_repository_branch"],
301-
lab=record_misc.get("runtime") if record_misc else None,
303+
lab=(
304+
record_misc.get("runtime", UNKNOWN_STRING)
305+
if record_misc
306+
else UNKNOWN_STRING
307+
),
302308
)
303309

304310
task.append(test_history_item)

‎backend/kernelCI_app/helpers/treeDetails.py‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,9 @@ def get_current_row_data(current_row: dict) -> dict:
112112
)
113113
current_row_data["test_platform"] = environment_misc.get("platform")
114114
test_misc = sanitize_dict(current_row_data["test_misc"])
115-
test_runtime_lab = test_misc.get("runtime") if test_misc is not None else None
115+
test_runtime_lab = UNKNOWN_STRING
116+
if test_misc is not None:
117+
test_runtime_lab = test_misc.get("runtime", UNKNOWN_STRING)
116118

117119
if current_row_data["test_status"] is None:
118120
current_row_data["test_status"] = NULL_STATUS
@@ -286,6 +288,9 @@ def decide_if_is_build_filtered_out(instance, row_data):
286288
build_duration = row_data["build_duration"]
287289
incident_test_id = row_data["incident_test_id"]
288290
build_origin = row_data["build_origin"]
291+
build_lab = UNKNOWN_STRING
292+
if row_data.get("build_misc") is not None:
293+
build_lab = row_data["build_misc"].get("lab", UNKNOWN_STRING)
289294

290295
is_build_filtered_out = instance.filters.is_build_filtered_out(
291296
build_status=build_status,
@@ -294,6 +299,7 @@ def decide_if_is_build_filtered_out(instance, row_data):
294299
issue_version=issue_version,
295300
incident_test_id=incident_test_id,
296301
build_origin=build_origin,
302+
build_lab=build_lab,
297303
)
298304
return is_build_filtered_out
299305

@@ -306,6 +312,7 @@ def decide_if_is_boot_filtered_out(instance, row_data):
306312
test_path = row_data["test_path"]
307313
incident_test_id = row_data["incident_test_id"]
308314
origin = row_data["test_origin"]
315+
lab = row_data["history_item"].get("lab", UNKNOWN_STRING)
309316

310317
return instance.filters.is_boot_filtered_out(
311318
duration=test_duration,
@@ -315,6 +322,7 @@ def decide_if_is_boot_filtered_out(instance, row_data):
315322
status=test_status,
316323
incident_test_id=incident_test_id,
317324
origin=origin,
325+
lab=lab,
318326
)
319327

320328

@@ -337,6 +345,7 @@ def decide_if_is_test_filtered_out(instance, row_data):
337345
test_path = row_data["test_path"]
338346
incident_test_id = row_data["incident_test_id"]
339347
origin = row_data["test_origin"]
348+
lab = row_data["history_item"].get("lab", UNKNOWN_STRING)
340349

341350
return instance.filters.is_test_filtered_out(
342351
duration=test_duration,
@@ -346,6 +355,7 @@ def decide_if_is_test_filtered_out(instance, row_data):
346355
status=test_status,
347356
incident_test_id=incident_test_id,
348357
origin=origin,
358+
lab=lab,
349359
)
350360

351361

@@ -371,6 +381,8 @@ def process_test_summary(instance, row_data):
371381
test_platform = row_data["test_platform"]
372382
test_error = row_data["test_error"]
373383
test_environment_compatible = row_data["test_environment_compatible"]
384+
test_origin = row_data["test_origin"]
385+
test_lab = row_data["history_item"].get("lab", UNKNOWN_STRING)
374386

375387
instance.testStatusSummary[test_status] = (
376388
instance.testStatusSummary.get(test_status, 0) + 1
@@ -402,20 +414,27 @@ def process_test_summary(instance, row_data):
402414
instance.testEnvironmentMisc[test_platform][test_status] += 1
403415

404416
increment_test_origin_summary(
405-
test_origin=row_data["test_origin"],
417+
test_origin=test_origin,
406418
test_status=test_status,
407419
origin_summary=instance.test_summary["origins"],
408420
)
409421

422+
if instance.test_summary_typed.labs.get(test_lab) is None:
423+
instance.test_summary_typed.labs[test_lab] = StatusCount()
424+
instance.test_summary_typed.labs[test_lab].increment(test_status)
410425

411-
def process_boots_summary(instance, row_data):
426+
427+
# TODO: use types instead of all dicts, receive specific fields instead of entire row_data
428+
def process_boots_summary(instance, row_data: dict[str, Any]) -> None:
412429
test_status = row_data["test_status"]
413430
build_config = row_data["build_config_name"]
414431
build_arch = row_data["build_architecture"]
415432
build_compiler = row_data["build_compiler"]
416433
test_platform = row_data["test_platform"]
417434
test_error = row_data["test_error"]
418435
test_environment_compatible = row_data["test_environment_compatible"]
436+
test_origin = row_data["test_origin"]
437+
test_lab = row_data["history_item"].get("lab", UNKNOWN_STRING)
419438

420439
instance.bootStatusSummary[test_status] = (
421440
instance.bootStatusSummary.get(test_status, 0) + 1
@@ -447,11 +466,15 @@ def process_boots_summary(instance, row_data):
447466
instance.bootEnvironmentMisc[test_platform][test_status] += 1
448467

449468
increment_test_origin_summary(
450-
test_origin=row_data["test_origin"],
469+
test_origin=test_origin,
451470
test_status=test_status,
452471
origin_summary=instance.boot_summary["origins"],
453472
)
454473

474+
if instance.boot_summary_typed.labs.get(test_lab) is None:
475+
instance.boot_summary_typed.labs[test_lab] = StatusCount()
476+
instance.boot_summary_typed.labs[test_lab].increment(test_status)
477+
455478

456479
def process_filters(instance, row_data: dict) -> None:
457480
issue_id = row_data["issue_id"]
@@ -464,6 +487,8 @@ def process_filters(instance, row_data: dict) -> None:
464487
instance.global_architectures.add(row_data["build_architecture"])
465488
instance.global_compilers.add(row_data["build_compiler"])
466489
instance.unfiltered_origins["build"].add(row_data["build_origin"])
490+
if (build_misc := row_data["build_misc"]) is not None:
491+
instance.unfiltered_labs["build"].add(build_misc.get("lab", UNKNOWN_STRING))
467492

468493
build_issue_id, build_issue_version, is_build_issue = (
469494
should_increment_build_issue(
@@ -495,10 +520,12 @@ def process_filters(instance, row_data: dict) -> None:
495520
if is_boot(row_data["test_path"]):
496521
issue_set = instance.unfiltered_boot_issues
497522
origin_set = instance.unfiltered_origins["boot"]
523+
lab_set = instance.unfiltered_labs["boot"]
498524
flag_tab: PossibleTabs = "boot"
499525
else:
500526
issue_set = instance.unfiltered_test_issues
501527
origin_set = instance.unfiltered_origins["test"]
528+
lab_set = instance.unfiltered_labs["test"]
502529
flag_tab: PossibleTabs = "test"
503530

504531
is_failed_test = row_data["test_status"] == FAIL_STATUS
@@ -513,3 +540,5 @@ def process_filters(instance, row_data: dict) -> None:
513540
)
514541

515542
origin_set.add(row_data["test_origin"])
543+
if (history_item := row_data.get("history_item")) is not None:
544+
lab_set.add(history_item.get("lab", UNKNOWN_STRING))

‎backend/kernelCI_app/queries/tree.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,7 @@ def get_tree_commit_history(
597597
t.environment_compatible,
598598
t.environment_misc,
599599
t.origin,
600+
t.misc->>'runtime' AS test_lab,
600601
b.id AS build_id,
601602
b.misc AS build_misc,
602603
t.id AS test_id,

‎backend/kernelCI_app/tests/integrationTests/hardwareDetailsSummary_test.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ def test_invalid_filters(invalid_filters_input):
431431
"DONE": 0,
432432
},
433433
"unknown_issues": 0,
434+
"labs": {},
434435
}
435436

436437
empty_build = {
@@ -448,6 +449,7 @@ def test_invalid_filters(invalid_filters_input):
448449
"DONE": 0,
449450
},
450451
"unknown_issues": 0,
452+
"labs": {},
451453
}
452454

453455
empty_summary = {

‎backend/kernelCI_app/tests/integrationTests/treeDetailsSummary_test.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ def test_invalid_filters(invalid_filters_input):
404404
"DONE": 0,
405405
},
406406
"unknown_issues": 0,
407+
"labs": {},
407408
}
408409

409410
empty_build = {
@@ -421,6 +422,7 @@ def test_invalid_filters(invalid_filters_input):
421422
"DONE": 0,
422423
},
423424
"unknown_issues": 0,
425+
"labs": {},
424426
}
425427

426428
empty_summary = {

‎backend/kernelCI_app/tests/unitTests/helpers/fixtures/hardware_details_data.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ def create_test_summary(**overrides):
114114
failed_platforms={"x86_64", "arm64"},
115115
environment_compatible={"hardware1": StatusCount()},
116116
environment_misc={"x86_64": StatusCount()},
117+
labs={},
117118
)
118119

119120
for key, value in overrides.items():

‎backend/kernelCI_app/tests/unitTests/helpers/fixtures/tree_details_data.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ def create_summary_row_data(**overrides):
8181
"test_error": "Test error",
8282
"test_environment_compatible": "hardware1",
8383
"test_origin": "test",
84+
"history_item": {
85+
"lab": "test_runtime_lab",
86+
},
8487
}
8588
base_data.update(overrides)
8689
return base_data
@@ -102,6 +105,10 @@ def create_filter_row_data(**overrides):
102105
"build_compiler": "gcc",
103106
"build_origin": "build_origin",
104107
"test_origin": "test_origin",
108+
"build_misc": {"lab": "lab1"},
109+
"history_item": {
110+
"lab": "test_runtime_lab",
111+
},
105112
}
106113
base_data.update(overrides)
107114
return base_data

‎backend/kernelCI_app/tests/unitTests/helpers/hardwareDetails_helpers_test.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,7 @@ def test_handle_test_summary(
617617
unknown_issues=0,
618618
fail_reasons={},
619619
failed_platforms=set(),
620+
labs={},
620621
)
621622

622623
issue_dict = {}
@@ -1317,6 +1318,7 @@ def test_format_issue_summary_for_response(self, mock_convert):
13171318
unknown_issues=0,
13181319
fail_reasons={},
13191320
failed_platforms=set(),
1321+
labs={},
13201322
)
13211323
tests_summary = TestSummary(
13221324
status=StatusCount(),
@@ -1327,6 +1329,7 @@ def test_format_issue_summary_for_response(self, mock_convert):
13271329
unknown_issues=0,
13281330
fail_reasons={},
13291331
failed_platforms=set(),
1332+
labs={},
13301333
)
13311334

13321335
issue_dicts = {

0 commit comments

Comments
 (0)