Cleanup

utils/allowlist.py

@@ -16,8 +16,6 @@ import datetime
 import requests
 import json
 import os
-import pandas
-import time
 import utils.pretty as ct
 import ijson
 import os
@@ -158,7 +156,6 @@ def listAllowlists(url):
         return choice, policiesnames, policyids
     #Need else and catch for upper bound
 
-
 def skipback(days):
     """
     Generate a MongoDB ObjectId for a given number of days ago from today.

utils/hashfunctions.py

@@ -12,14 +12,13 @@
 #
 #    You should have received a copy of the GNU Affero General Public License
 #    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import pandas as pd
 import requests
 import os
 import json
 import utils.pretty as ct
 
-
-
 def aggregateHashes(executions_json) -> pd.DataFrame:
     """
     Takes the executions,  aggregates all the data with sha256 as primary, then returns aggregated dataframe
@@ -41,7 +40,6 @@ def aggregateHashes(executions_json) -> pd.DataFrame:
 
     return agg_df
 
-
 def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
     """
     Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
@@ -103,7 +101,6 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
 
     return aug_df
 
-
 def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
     if untrusted_publishers is None:
         untrusted_publishers = []
@@ -146,9 +143,6 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
 
     return needsreview_df, approved_df, unapproved_df
 
-
-import pandas as pd
-
 def explode_and_deduplicate(df):
     df['sha256'] = df['sha256'].str.split(',')
     df = df.explode('sha256')
@@ -205,60 +199,3 @@ def destinationHashes(
     # Concatenate results
     df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
     return df_hashdestination
-
-
-
-"""
-
-def explode_and_deduplicate(df):
-    df['sha256'] = df['sha256'].str.split(',')
-    df = df.explode('sha256')
-    return df.drop_duplicates().reset_index(drop=True)
-
-def clean_sha256(df, column='sha256'):
-    df[column] = df[column].astype(str).str.strip("'[]\" ")
-    return df
-
-def destinationHashes(
-    df_approved_paths: pd.DataFrame,
-    df_approved_hashes: pd.DataFrame,
-    df_hashes_auto_approved: pd.DataFrame,
-    df_hashes_manually_approved: pd.DataFrame,
-):
-    # Deduplicate and explode all input DataFrames
-    df_approved_paths = explode_and_deduplicate(df_approved_paths)
-    df_approved_hashes = explode_and_deduplicate(df_approved_hashes)
-    df_hashes_auto_approved = explode_and_deduplicate(df_hashes_auto_approved)
-    df_hashes_manually_approved = explode_and_deduplicate(df_hashes_manually_approved)
-
-    # Clean sha256 values in all relevant DataFrames
-    df_approved_hashes = clean_sha256(df_approved_hashes)
-    df_hashes_auto_approved = clean_sha256(df_hashes_auto_approved)
-    df_hashes_manually_approved = clean_sha256(df_hashes_manually_approved)
-
-    # Create sets for faster lookup
-    auto_approved_sha256 = set(df_hashes_auto_approved['sha256'].values)
-    manually_approved_sha256 = set(df_hashes_manually_approved['sha256'].values)
-
-    # Debug: Print unmatched hashes
-    unmatched = set(df_approved_hashes['sha256']) - (auto_approved_sha256 | manually_approved_sha256)
-    print(f"Unmatched hashes: {unmatched}")
-
-    # Process df_approved_paths
-    df_paths = df_approved_paths.assign(
-        destination='Path Exclusion'
-    )[['sha256', 'description', 'destination', 'grouped_directory']]
-
-    # Process df_approved_hashes
-    df_hashes = df_approved_hashes.copy()
-    df_hashes['destination'] = df_hashes['sha256'].apply(
-        lambda x: 'Parent Policy Baseline' if x in auto_approved_sha256
-                  else ('Child Policy Allowlist' if x in manually_approved_sha256 else None)
-    )
-    df_hashes = df_hashes.dropna(subset=['destination'])
-    df_hashes = df_hashes.assign(grouped_directory=None)[['sha256', 'description', 'destination', 'grouped_directory']]
-
-    # Concatenate results
-    df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
-    return df_hashdestination
-"""

utils/pathfunctions.py

@@ -19,9 +19,6 @@ from itertools import chain
 import ast
 import re
 
-
-
-
 def split_path(path):
     parts = []
     while True:
@@ -64,7 +61,6 @@ def local_common_pass(paths, min_parts=3):
         results[path] = best
     return results
 
-
 def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3):
     dirs_series = df[col].astype(str).apply(os.path.dirname)
     first_pass = local_common_pass(dirs_series.tolist(), min_parts)
@@ -72,7 +68,6 @@ def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min
     df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]])
     return df
 
-
 def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant):
     """
     Compute longest common paths, group filepaths, write CSV for review.
@@ -88,8 +83,6 @@ def export_groups_for_review(df, col, group_col, min_number_in_group, path_lengt
 
     return filtered, df
 
-
-
 def mask_from_csv(df, csv_path, filepath_col):
     """
     Reads reviewed CSV of groups, keeps only files in approved groups.
@@ -120,7 +113,6 @@ def mask_from_csv(df, csv_path, filepath_col):
     remainder = df[~df[filepath_col].isin(approved_files)].copy()
     return remainder
 
-
 def filter_and_drop(approved, eligiblepaths, min_hashes):
     """
     Filters eligiblepaths to rows where all hashes are in approved,
@@ -136,7 +128,6 @@ def filter_and_drop(approved, eligiblepaths, min_hashes):
     
     return filtered
 
-
 def inspect_parquet(path):
     try:
         df = pd.read_parquet(path)
@@ -149,7 +140,6 @@ def inspect_parquet(path):
         return pd.DataFrame()
 
 
-
 def regulator(paths, case_insensitive=True):
     """
     Build a Python raw string regex that matches any of the given Windows path fragments.

utils/policyfunctions.py

@@ -25,9 +25,6 @@ def addHash(policy, hash):
 def addPath(policy, hash):
     print(f"Adding the following Path Exclusions to {policy}:")
     print(hash)
-
-
-
     
 def addHashReal(url, allowlistID, hashlist): 
     endpoint = url + '/v1/hash/application/add'