Paths fixed... need to double check the path exclusion format to feed api

utils/hashfunctions.py

@@ -19,6 +19,7 @@ import os
 import json
 import utils.pathfunctions as pathf
 import utils.pretty as ct
+import gc
 
 
 def aggregateHashes(executions_json) -> pd.DataFrame:
@@ -103,7 +104,7 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
 
     return aug_df
 
-def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list, pups: list):
+def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishers, pups: list):
     if untrusted_publishers is None: untrusted_publishers = []
     if pups is None: pups = []
 
@@ -126,14 +127,14 @@ def categorizeHashes(df: pd.DataFrame, threat_tolerance: int, untrusted_publishe
     mask_approved = (
         (
             (df["publisher"] != "Not Signed") &
-            ~df["publisher"].isin(untrusted_publishers) &
+            ~df["publisher"].str.contains(pathf.regulator(untrusted_publishers), case=False, na=False) &
             ~df["reputation_status"].isna() &
             ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         ) |
         (
             (df["publisher"] == "Not Signed") &
             ~df["reputation_flag"] &
-            ~df["publisher"].isin(untrusted_publishers) &
+            ~df["publisher"].str.contains(pathf.regulator(untrusted_publishers), case=False, na=False) &
             ~df["reputation_status"].isna() &
             ~df["description"].str.contains(pathf.regulator(pups), case=False, na=False)
         )
@@ -203,3 +204,21 @@ def destinationHashes(
     # Concatenate results
     df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
     return df_hashdestination
+
+def combineHashAndHist(path, first_policy, second_policy):
+
+    condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
+    df = pd.read_parquet(path)
+			
+	#Pull hash info for the entries in the needs approval table
+    df = pd.merge(condensed_combo, df, on='sha256', how='inner')
+    
+    #Rename Publisher, Keep and reorder columns we want
+    df = df.rename(columns={'publisher_x': 'publisher'})
+    df = df[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
+    df = df.sort_values(by='filename')
+
+    df.to_parquet(path, index=False)
+    del df
+    del condensed_combo
+    gc.collect()

utils/pathfunctions.py

@@ -15,73 +15,60 @@
 
 import pandas as pd
 import os
-from itertools import chain
 import ast
 import re
 
-def split_path(path):
-    parts = []
-    while True:
-        head, tail = os.path.split(path)
-        if tail:
-            parts.insert(0, tail)
-            path = head
-        else:
-            if head:
-                parts.insert(0, head)
-            break
-    return parts
+import os
+import pandas as pd
 
-def local_common_pass(paths, min_parts=3):
-    results = {}
-    paths_sorted = sorted(paths)
-    for i, path in enumerate(paths_sorted):
-        candidates = []
+import os
+import pandas as pd
 
-        if i > 0:
-            try:
-                candidates.append(os.path.commonpath([path, paths_sorted[i-1]]))
-            except ValueError:
-                # different drives, skip
-                pass
-        if i < len(paths_sorted) - 1:
-            try:
-                candidates.append(os.path.commonpath([path, paths_sorted[i+1]]))
-            except ValueError:
-                # different drives, skip
-                pass
+def split_filepaths_grouped(df, col="filename", group_parts=3, min_parts=3):
+    def clean_split(path):
+        parts = os.path.normpath(path).split(os.sep)
+        # Remove leading empty strings caused by UNC paths
+        parts = [p for p in parts if p]
+        return parts
 
-        best = path
-        best_len = 0
-        for c in candidates:
-            parts = split_path(c)
-            if len(parts) >= min_parts and len(parts) > best_len:
-                best = c
-                best_len = len(parts)
-        results[path] = best
-    return results
+    df = df.copy()
+    split_paths = df[col].apply(clean_split)
 
-def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3):
-    dirs_series = df[col].astype(str).apply(os.path.dirname)
-    first_pass = local_common_pass(dirs_series.tolist(), min_parts)
-    second_pass = local_common_pass(list(first_pass.values()), min_parts)
-    df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]])
-    return df
+    # Filter out paths with fewer than `min_parts` components
+    df = df[split_paths.apply(lambda parts: len(parts) >= min_parts)].copy()
+    split_paths = split_paths[df.index]  # Update split_paths to match filtered df
 
-def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant):
-    """
-    Compute longest common paths, group filepaths, write CSV for review.
-    """
-    df = df.drop_duplicates(subset=[col], keep='first')
-    df = add_longest_common_two_local(df, col=col, new_col=group_col)
-    grouped = df.groupby(group_col)[col].apply(list).reset_index()
-    grouped = grouped.sort_values(by=col)
-    print("Before filtering:", len(grouped))
-    grouped = grouped[grouped[col].apply(lambda x: len(x) >= min_number_in_group)]
-    filtered = grouped[grouped[group_col].apply(lambda x: len(os.path.normpath(x).split(os.sep)) >= path_length_constant)]
-    print("After filtering:", len(grouped))
+    df["group_key"] = split_paths.apply(lambda parts: os.sep.join(parts[:group_parts]))
+    grouped = df.groupby("group_key")
+    new_rows = []
 
-    return filtered, df
+    for _, group_df in grouped:
+        paths = group_df[col].tolist()
+        split_parts = [clean_split(p) for p in paths]
+
+        def longest_common_prefix(paths):
+            if not paths:
+                return []
+            prefix = paths[0]
+            for path in paths[1:]:
+                prefix = [a for a, b in zip(prefix, path) if a == b]
+                if not prefix:
+                    break
+            return prefix
+
+        common_prefix = longest_common_prefix(split_parts)
+        prefix_str = os.sep.join(common_prefix)
+
+        for i, parts in enumerate(split_parts):
+            filename = parts[-1]
+            middle = os.sep.join(parts[len(common_prefix):-1]) if len(parts) > len(common_prefix) + 1 else ""
+            row = group_df.iloc[i].copy()
+            row["longestcfp"] = prefix_str
+            row["middle"] = middle
+            row["filename_only"] = filename
+            new_rows.append(row)
+
+    return pd.DataFrame(new_rows).drop(columns=["group_key"])
 
 def mask_from_csv(df, csv_path, filepath_col):
     """