Fixed some logic, added functions to list allowlist, and category. Category list function will need work, the subcategories are not being parsed at this time
This commit is contained in:
+18
-4
@@ -81,24 +81,38 @@ def export_groups_for_review(df, col="filename_x", group_col="longestcfp"):
|
||||
return grouped, df
|
||||
|
||||
|
||||
|
||||
def mask_from_csv(df, csv_path, filepath_col):
|
||||
"""
|
||||
Reads reviewed CSV of groups, keeps only files in approved groups.
|
||||
"""
|
||||
review_df = pd.read_csv(csv_path)
|
||||
# Convert string representation of lists back to actual lists
|
||||
review_df[filepath_col] = review_df[filepath_col].apply(ast.literal_eval)
|
||||
|
||||
|
||||
def parse_paths(val):
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
# Try to parse as a list
|
||||
parsed = ast.literal_eval(val)
|
||||
# If it's not a list, wrap it
|
||||
return parsed if isinstance(parsed, list) else [parsed]
|
||||
except (ValueError, SyntaxError):
|
||||
# If parsing fails, treat it as a single path
|
||||
return [val]
|
||||
return [val]
|
||||
|
||||
review_df[filepath_col] = review_df[filepath_col].apply(parse_paths)
|
||||
|
||||
# Flatten all approved file paths into a set for masking
|
||||
approved_files = set()
|
||||
for paths in review_df[filepath_col]:
|
||||
approved_files.update(paths)
|
||||
|
||||
|
||||
# Keep only rows in df that are in approved_files
|
||||
masked_df = df[df[filepath_col].isin(approved_files)].copy()
|
||||
remainder = df[~df[filepath_col].isin(approved_files)].copy()
|
||||
return masked_df, remainder
|
||||
|
||||
|
||||
def filter_and_drop(approved, eligiblepaths, min_hashes):
|
||||
"""
|
||||
Filters eligiblepaths to rows where all hashes are in approved,
|
||||
|
||||
Reference in New Issue
Block a user