Paths fixed... need to double check the path exclusion format to feed api

utils/hashfunctions.py

@@ -19,6 +19,7 @@ import os
 import json
 import utils.pathfunctions as pathf
 import utils.pretty as ct
+import gc
 
 
 def aggregateHashes(executions_json) -> pd.DataFrame:
@@ -103,7 +104,7 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
 
     return aug_df
 
-def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list, pups: list):
+def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishers, pups: list):
     if untrusted_publishers is None: untrusted_publishers = []
     if pups is None: pups = []
 
@@ -126,14 +127,14 @@ def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishe
     mask_approved = (
         (
             (df["publisher"] != "Not Signed") &
-            ~df["publisher"].isin(untrusted_publishers) &
+            ~df["publisher"].str.contains(pathf.regulator(untrusted_publishers), case=False, na=False) &
             ~df["reputation_status"].isna() &
             ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         ) |
         (
             (df["publisher"] == "Not Signed") &
             ~df["reputation_flag"] &
-            ~df["publisher"].isin(untrusted_publishers) &
+            ~df["publisher"].str.contains(pathf.regulator(untrusted_publishers), case=False, na=False) &
             ~df["reputation_status"].isna() &
             ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         )
@@ -203,3 +204,21 @@ def destinationHashes(
     # Concatenate results
     df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
     return df_hashdestination
+
+def combineHashAndHist(path, first_policy, second_policy):
+
+    condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
+    df = pd.read_parquet(path)
+			
+	#Pull hash info for the entries in the needs approval table
+    df = pd.merge(condensed_combo, df, on='sha256', how='inner')
+    
+    #Rename Publisher, Keep and reorder columns we want
+    df = df.rename(columns={'publisher_x': 'publisher'})
+    df = df[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
+    df = df.sort_values(by='filename')
+
+    df.to_parquet(path, index=False)
+    del df
+    del condensed_combo
+    gc.collect()