Fixed some logic, added functions to list allowlist, and category. Category list function will need work, the subcategories are not being parsed at this time

This commit is contained in:
=
2025-08-28 09:06:25 -04:00
parent f94c3e432e
commit e51ed0543a
4 changed files with 96 additions and 25 deletions
+18 -4
View File
@@ -81,24 +81,38 @@ def export_groups_for_review(df, col="filename_x", group_col="longestcfp"):
return grouped, df
def mask_from_csv(df, csv_path, filepath_col):
"""
Reads reviewed CSV of groups, keeps only files in approved groups.
"""
review_df = pd.read_csv(csv_path)
# Convert string representation of lists back to actual lists
review_df[filepath_col] = review_df[filepath_col].apply(ast.literal_eval)
def parse_paths(val):
if isinstance(val, str):
try:
# Try to parse as a list
parsed = ast.literal_eval(val)
# If it's not a list, wrap it
return parsed if isinstance(parsed, list) else [parsed]
except (ValueError, SyntaxError):
# If parsing fails, treat it as a single path
return [val]
return [val]
review_df[filepath_col] = review_df[filepath_col].apply(parse_paths)
# Flatten all approved file paths into a set for masking
approved_files = set()
for paths in review_df[filepath_col]:
approved_files.update(paths)
# Keep only rows in df that are in approved_files
masked_df = df[df[filepath_col].isin(approved_files)].copy()
remainder = df[~df[filepath_col].isin(approved_files)].copy()
return masked_df, remainder
def filter_and_drop(approved, eligiblepaths, min_hashes):
"""
Filters eligiblepaths to rows where all hashes are in approved,