Fixed several logic issues / generation of secondary paths. Cleaned up Policy Prep text to be accurate to current functionality. Fixed issue with going back from policy prep menu. Switched to logging.dict from standard logging config.

flows/prepPolicy.py

@@ -132,6 +132,7 @@ def buildPathsandPublishers(selected_policies: List[Policy], split):
     all_approved_hashes = pd.DataFrame()
     path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_approved_executions.csv"
     path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_needs_review_executions.csv"
+    path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int)
 
     if os.path.exists(path1):
         df1 = pd.read_csv(path1)
@@ -154,29 +155,31 @@ def buildPathsandPublishers(selected_policies: List[Policy], split):
         else:
             logger.warning("Warning: 'filename' column not found in concatenated DataFrame.")
 
-    if not all_approved_hashes.empty:
+    if not all_approved_hashes.empty and path_exclusion_constant:
+
         primary_path_exclusions = calculatePath(
-            all_approved_hashes,
+            all_approved_hashes, path_exclusion_constant,
             split,
         )
         remaining_hashes = all_approved_hashes[
             ~all_approved_hashes["sha256"].isin(primary_path_exclusions["sha256"])
         ]
         secondary_path_exclusions = calculatePath(
-            remaining_hashes, split
+            remaining_hashes,(path_exclusion_constant - 1), split
         )
         remaining_hashes = remaining_hashes[
             ~remaining_hashes["sha256"].isin(secondary_path_exclusions["sha256"])
         ]
         dataframes = {
+            "all_approved_hashes" : all_approved_hashes,
             "primary_Paths": primary_path_exclusions,
             "secondary_Paths": secondary_path_exclusions,
-            "hashes_to_add": remaining_hashes,
+            "hashes_not_approvable_by_path": remaining_hashes
         }
         logger.debug("Preparing to sort dataframes")
         for name, df in dataframes.items():
             logger.debug(f" DataFrame headers: {list(df.columns)}")
-            if name == "hashes_to_add":   df.sort_values(by="filename", inplace=True)
+            if "hashes" in name :   df.sort_values(by="filename", inplace=True)
             else: df.sort_values(by="longestcfp", inplace=True)
           
             df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_{name}.csv", index=False)
@@ -193,6 +196,8 @@ def buildPathsandPublishers(selected_policies: List[Policy], split):
             publist = publist[["publisher"]]
             publist.sort_values(by="publisher", inplace=True)
             publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_publishers.csv", index=False)
+    else: 
+        logger.debug("Approved Hashes list appears empty")
 
 def buildPreflights(selected_policies: List[Policy]):
     working_dir = load_env("WORKING_DIR")
@@ -202,17 +207,13 @@ def buildPreflights(selected_policies: List[Policy]):
     approved_hashes = pd.DataFrame()
     approved_publishers = pd.DataFrame()
 
-    hash = f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv"
+    hash = f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_all_approved_hashes.csv"
     path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv"
     path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_secondary_Paths.csv"
     publishers = f"{working_dir}\\Approved\\{selected_policies[0].name}_publishers.csv"
 
-    if os.path.exists(hash):
-        approved_hashes = pd.read_csv(hash)
-
-    else:
-        logger.warning(f"File not found: {hash}")
 
+    #Read in and combine the two path generations
     if os.path.exists(path1):
         df1 = pd.read_csv(path1)
     else:
@@ -228,6 +229,19 @@ def buildPreflights(selected_policies: List[Policy]):
         approved_paths = pd.DataFrame()
     else:
         approved_paths = pd.concat([df1, df2], ignore_index=True)
+    
+    approved_paths = approved_paths.drop_duplicates(subset="longestcfp", keep ="first")
+
+    #We create a list of hashes that are left over if we exclude the ones that are covered by the path exclusions.
+    if os.path.exists(hash):
+        hashes = pd.read_csv(hash)
+        approved_hashes = hashes[~hashes['filename'].isin(approved_paths['longestcfp'])]
+
+        approved_hashes = approved_hashes.drop_duplicates(subset="sha256", keep ="first")
+
+    else:
+            logger.warning(f"File not found: {hash}")
+
 
     if os.path.exists(publishers):
         approved_publishers = pd.read_csv(publishers)
@@ -246,8 +260,7 @@ def buildPreflights(selected_policies: List[Policy]):
         df.to_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_{name}.csv", index=False)
         formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{selected_policies[0].name}_{name}.html")
 
-def splitFilepathsGrouped(df, col="filename"):
-    path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int)
+def splitFilepathsGrouped(df, path_exclusion_constant, col="filename"):
     min_files_for_path = get_protected_value("MIN_FILES_FOR_PATH", cast_type= int)
 
     def clean_split(path):
@@ -307,7 +320,7 @@ def splitFilepathsGrouped(df, col="filename"):
 
     return pd.DataFrame(new_rows).drop(columns=["group_key"])
 
-def calculatePath(approved_hashes, split):
+def calculatePath(approved_hashes, path_exclusion_constant, split):
     if split:
         dfs_by_policy = [group for _, group in approved_hashes.groupby("policy")]
     else:
@@ -319,7 +332,7 @@ def calculatePath(approved_hashes, split):
     processed_dfs = []
 
     for df in dfs_by_policy:
-        haslcp = splitFilepathsGrouped(df, "filename")
+        haslcp = splitFilepathsGrouped(df, path_exclusion_constant, "filename")
         haslcp = haslcp.drop_duplicates()
 
         forbidden = regulator(badpathparts, True)