Added Pup Filter and Path Dedupe

utils/hashfunctions.py

@@ -17,8 +17,10 @@ import pandas as pd
 import requests
 import os
 import json
+import utils.pathfunctions as pathf
 import utils.pretty as ct
 
+
 def aggregateHashes(executions_json) -> pd.DataFrame:
     """
     Takes the executions,  aggregates all the data with sha256 as primary, then returns aggregated dataframe
@@ -101,11 +103,9 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
 
     return aug_df
 
-def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
-    if untrusted_publishers is None:
-        untrusted_publishers = []
-
-    df = aug_df.copy()
+def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list, pups: list):
+    if untrusted_publishers is None: untrusted_publishers = []
+    if pups is None: pups = []
 
     def reputationtool(row):
         val = row["reputation_scannermatch"]
@@ -127,13 +127,15 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
         (
             (df["publisher"] != "Not Signed") &
             ~df["publisher"].isin(untrusted_publishers) &
-            ~df["reputation_status"].isna()
+            ~df["reputation_status"].isna() &
+            ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         ) |
         (
             (df["publisher"] == "Not Signed") &
             ~df["reputation_flag"] &
             ~df["publisher"].isin(untrusted_publishers) &
-            ~df["reputation_status"].isna()
+            ~df["reputation_status"].isna() &
+            ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         )
     )
 
@@ -143,6 +145,8 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
 
     return needsreview_df, approved_df, unapproved_df
 
+
+
 def explode_and_deduplicate(df):
     df['sha256'] = df['sha256'].str.split(',')
     df = df.explode('sha256')