From b41e56d84acaf63ddf19ef62e37370644ad638cc Mon Sep 17 00:00:00 2001
From: = <=>
Date: Tue, 2 Sep 2025 22:36:11 -0400
Subject: [PATCH] Better Path grouping

---
 AirlockTools.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/AirlockTools.py b/AirlockTools.py
index 050bdb4..dea28dd 100644
--- a/AirlockTools.py
+++ b/AirlockTools.py
@@ -294,7 +294,7 @@ def menu_prepare_to_enforce():
 					exe1 = pd.read_parquet(f"parquet\\execution_history_{first_policy}.parquet")
 					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{first_policy}.parquet")
 					if not exe1.empty:
-						condensed_exe1 = exe1.groupby('sha256').agg(lambda x: list(set(x.tolist()))).reset_index()
+						print()
 					else:
 						print("⚠️ First dataframe is empty.")
 				except Exception as e:
@@ -304,26 +304,20 @@ def menu_prepare_to_enforce():
 					exe2 = pd.read_parquet(f"parquet\\execution_history_{second_policy}.parquet")
 					utils.pathfunctions.inspect_parquet(f"parquet\\execution_history_{second_policy}.parquet")
 					if not exe2.empty:
-						condensed_exe2 = exe2.groupby('sha256').agg(lambda x: list(set(x.tolist()))).reset_index()
-
+						print()
 					else:
 						print("⚠️ Second dataframe is empty.")
 				except Exception as e:
 					print(f"❌ Error reading second Parquet file: {e}")
 
 				if not exe1.empty and not exe2.empty:
-					condensed_combo = pd.concat([condensed_exe1, condensed_exe2], ignore_index=True)
-					# Group by 'sha256' and merge list columns manually
-					condensed_combo = condensed_combo.groupby('sha256').agg(
-						lambda col: list(set([item for sublist in col if isinstance(sublist, list) for item in sublist]))
-					).reset_index()
-
+					condensed_combo = pd.concat([exe1, exe2], ignore_index=True)
 
 					print(f"✅ Combined {len(condensed_combo)} hashes.")
 				elif exe1.empty:
-					condensed_combo = condensed_exe2
+					condensed_combo = exe2
 				elif exe2.empty:
-					condensed_combo = condensed_exe1
+					condensed_combo = exe1
 				else:
 					print("⚠️ No valid dataframes to combine.")
 
@@ -466,8 +460,16 @@ def menu_prepare_to_enforce():
 		
 					forbidden = utils.pathfunctions.regulator(badpathparts, True)
 					forbidden_lcfp = grouped_df_view["longestcfp"].str.contains(forbidden, na=False)
-					grouped_df_view = grouped_df_view[~forbidden_lcfp]
-					print(ct.colorText(f"Removing forbidden filepaths for path exceptions","green"))
+
+					# Make a real DataFrame copy before modifying
+					grouped_df_view = grouped_df_view[~forbidden_lcfp].copy()
+
+					print(ct.colorText("Removing forbidden filepaths for path exceptions", "green"))
+
+					for col in grouped_df_view.columns:
+							if grouped_df_view[col].apply(lambda x: isinstance(x, list)).all():
+								grouped_df_view[col] = grouped_df_view[col].apply(deduplicate_list)
+
 
 					grouped_df_view.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet", index=False)
 					grouped_df_view.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv", index=False)