Cleaned up menu by functionalizing

utils/pathfunctions.py

@@ -12,11 +12,14 @@
 #
 #    You should have received a copy of the GNU Affero General Public License
 #    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import pandas as pd
-import os
 import ast
+import gc
+import os
+import pandas as pd
 import re
+import utils.pathfunctions as pathf
+import utils.pretty as ct
+from AirlockTools import tryToReadCSV
 
 
 def split_filepaths_grouped(df, col="filename", group_parts=3, min_parts=3):
@@ -133,3 +136,49 @@ def regulator(paths, case_insensitive=True):
     print(f"Regulator is providing: {pattern}")
     return pattern
 
+def generatePathReview(first_policy, second_policy, badpathparts, min_files_for_path):
+
+    if not os.path.exists(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet"):
+
+        df1 = tryToReadCSV(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv")
+        df2 = tryToReadCSV(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv")
+
+        all_approved_hashes = pd.concat([df1 , df2], ignore_index=True).sort_values(by=['filename'])
+
+
+
+        print(ct.colorText(f"Approved hash lists have been combined","green"))
+
+        all_approved_hashes.to_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet", index=False)
+        del all_approved_hashes
+        gc.collect()
+
+    if not os.path.exists(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet"):
+        all_approved_hashes = pd.read_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet")
+        print(ct.colorText(f"Beginning calculating longest common filepaths for path exceptions","green"))
+        
+        haslcp = pathf.split_filepaths_grouped(all_approved_hashes)
+        haslcp.drop_duplicates()
+
+        forbidden = pathf.regulator(badpathparts, True)
+        forbidden_lcfp = haslcp["longestcfp"].str.contains(forbidden, na=False)
+
+        
+        print(ct.colorText("Removing forbidden filepaths for path exceptions", "green"))
+
+        # Make a real DataFrame copy before modifying
+        lcp_not_forbidden = haslcp[~forbidden_lcfp].copy()
+
+        #For the review, drop down to only the columns we care, and then group by the commmon file path, consolidating and dropping dupes
+        lcp_not_forbidden_review = lcp_not_forbidden[['longestcfp', 'middle', 'filename_only', 'sha256']]				
+
+        # Count unique sha256 per longestcfp
+        unique_sha_counts = lcp_not_forbidden_review.groupby('longestcfp')['sha256'].nunique().reset_index()
+        unique_sha_counts.columns = ['longestcfp', 'unique_sha256_count']
+
+        # Merge the count back into the original DataFrame
+        lcp_not_forbidden_review = lcp_not_forbidden_review.merge(unique_sha_counts, on='longestcfp', how='left')
+        lcp_not_forbidden_review = lcp_not_forbidden_review[lcp_not_forbidden_review['unique_sha256_count'] >= min_files_for_path]
+        
+        lcp_not_forbidden_review.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet",index=False)
+        lcp_not_forbidden_review.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv",index=False)