Sharing

AirlockTools.py

@@ -47,15 +47,38 @@ def menu():
 			categorized[0].to_html("needsreview.html", index=False)
 			categorized[1].to_html("approved.html", index=False)
 			categorized[2].to_html("remaining.html", index=False)
+
 		if choice == '4':
 			html_file = "augmentedlist.html"
 			augmented_df = pd.read_html(html_file)
 			print(augmented_df)
 
 			combined_df = pd.concat(augmented_df, ignore_index=True)
-			test = utils.pathfunctions.filepathInitalGroup(combined_df)
+			path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(combined_df)
-			test.to_html("testgroup2.html", index=False)
+			path_eligible.to_html("EligblePaths.html", index=False)
+			path_ineligible.to_html("IneligiblePaths.html",index=False)
 		
+		if choice == '5':
+			
+			executionhist = utils.allowlist.allowlistexechistories(url,True)
+			print(executionhist)
+			aggregated = utils.hashfunctions.aggregateHashes(executionhist)
+			print(aggregated)
+			augmented = utils.hashfunctions.augmentAggregatedHashes(url,aggregated)
+			print(augmented)
+			augmented.to_html("augmentedlist.html", index=False)
+			html_file = "augmentedlist.html"
+			augmented_df = pd.read_html(html_file)
+			combined_df = pd.concat(augmented_df, ignore_index=True)
+			path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(combined_df)
+			badpublisherlist = []
+			categorized = utils.hashfunctions.categorizeHashes(augmented, 5, badpublisherlist)
+			categorized[0].to_html("needsreview.html", index=False)
+			categorized[1].to_html("approved.html", index=False)
+			categorized[2].to_html("remaining.html", index=False)
+			path_eligible, path_ineligible = utils.pathfunctions.filepathInitialGroup(combined_df)
+			path_eligible.to_html("EligblePaths.html", index=False)
+			path_ineligible.to_html("IneligiblePaths.html",index=False)
 
 
 if __name__ == "__main__":

utils/pathfunctions.py

@@ -1,39 +1,100 @@
 import pandas as pd
 import os
+from itertools import chain
 
+def filepathInitialGroup(df: pd.DataFrame):
+    original_columns = df.columns.tolist()
 
-def filepathInitalGroup(df: pd.DataFrame) -> pd.DataFrame:
+    # Step 1: Split comma-separated filepaths into lists
-    import os
-    import pandas as pd
-    from itertools import chain
-
-    # 1. Split comma-separated filenames into lists
     df["filename_x"] = df["filename_x"].str.split(",")
 
-    # 2. Explode so each filename has its own row
+    # Step 2: Explode the list so each filepath becomes its own row
     df = df.explode("filename_x", ignore_index=True)
 
-    # 3. Strip whitespace from filenames
+    # Step 3: Clean up whitespace
     df["filename_x"] = df["filename_x"].str.strip()
 
-    # 4. Split into directory and filename
+    # Step 4: Extract directory and filename from each filepath
     df["directory"] = df["filename_x"].apply(lambda x: os.path.dirname(x) if pd.notna(x) else "")
     df["filename"] = df["filename_x"].apply(lambda x: os.path.basename(x) if pd.notna(x) else "")
 
-    # 5. Drop the original column
+    # Step 5: Drop the original raw filepath column
     df = df.drop(columns=["filename_x"])
 
-    # 6. Ensure all columns are lists (except directory, which is the key)
+    # Helper functions for path manipulation
+    def get_parts(path):
+        return path.strip("\\").split("\\")
+
+    def join_parts(parts):
+        return "\\".join(parts)
+
+    def longest_common_prefix(paths):
+        split_paths = [get_parts(p) for p in paths]
+        min_len = min(len(p) for p in split_paths)
+        prefix = []
+        for i in range(min_len):
+            current = split_paths[0][i]
+            if all(p[i] == current for p in split_paths):
+                prefix.append(current)
+            else:
+                break
+        return join_parts(prefix)
+
+    # Step 6: Group directories by shared prefix using custom logic
+    directories = df["directory"].tolist()
+    groups = []
+    used = set()
+
+    for i, path in enumerate(directories):
+        if path in used:
+            continue
+        group = [path]
+        parts_i = get_parts(path)
+        for j in range(i + 1, len(directories)):
+            parts_j = get_parts(directories[j])
+            common = os.path.commonprefix([parts_i, parts_j])
+            if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
+                group.append(directories[j])
+                used.add(directories[j])
+            elif len(common) == len(parts_i) - 1 and len(parts_i) > 3:
+                group.append(directories[j])
+                used.add(directories[j])
+        used.add(path)
+        groups.append(group)
+
+    # Step 7: Map each original directory to its grouped prefix
+    prefix_map = {dir: longest_common_prefix(group) for group in groups for dir in group}
+    df["grouped_directory"] = df["directory"].map(prefix_map)
+
+    # Step 8: Group the DataFrame by grouped_directory
+    aggregation = {col: (lambda x: list(x)) for col in original_columns if col not in ["filename_x"]}
+    aggregation.update({
+        "directory": lambda x: list(x),
+        "filename": lambda x: list(x)
+    })
+
+    grouped_df = df.groupby("grouped_directory", as_index=False).agg(aggregation)
+
+    # Step 9: Split into eligible and ineligible paths based on depth
+    grouped_df["depth"] = grouped_df["grouped_directory"].apply(lambda x: len(get_parts(x)))
+    path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
+    path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])
+
+    # Step 10: Move entries from eligible to ineligible if grouped_directory contains 'C:\Users' or 'c$\Users'
+    mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users)")
+    move_to_ineligible = path_eligible[mask]
+    path_eligible = path_eligible[~mask]
+    path_ineligible = pd.concat([path_ineligible, move_to_ineligible], ignore_index=True)
+
+    # Step 11: Deduplicate list elements in all columns
+    def deduplicate_lists(df):
         for col in df.columns:
-        if col != "directory":
+            if df[col].apply(lambda x: isinstance(x, list)).all():
-            df[col] = df[col].apply(lambda x: x if isinstance(x, list) else [x])
+                df[col] = df[col].apply(lambda x: list({str(item): item for item in chain.from_iterable(x if isinstance(x[0], list) else [x])}.values()))
-
-    # 7. Combine rows with the same directory, flatten lists, deduplicate
-    def combine_lists(series):
-        flat = list(chain.from_iterable(series))
-        # deduplicate while preserving order
-        return list(dict.fromkeys(flat))
-
-    df = df.groupby("directory", as_index=False).agg(combine_lists)
-
         return df
+
+
+    path_eligible = deduplicate_lists(path_eligible)
+    path_ineligible = deduplicate_lists(path_ineligible)
+
+    return path_eligible, path_ineligible