diff --git a/utils/allowlist.py b/utils/allowlist.py index f63202a..9eeb0ec 100644 --- a/utils/allowlist.py +++ b/utils/allowlist.py @@ -16,8 +16,6 @@ import datetime import requests import json import os -import pandas -import time import utils.pretty as ct import ijson import os @@ -158,7 +156,6 @@ def listAllowlists(url): return choice, policiesnames, policyids #Need else and catch for upper bound - def skipback(days): """ Generate a MongoDB ObjectId for a given number of days ago from today. diff --git a/utils/hashfunctions.py b/utils/hashfunctions.py index 1a7810b..00d8c1c 100644 --- a/utils/hashfunctions.py +++ b/utils/hashfunctions.py @@ -12,14 +12,13 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + import pandas as pd import requests import os import json import utils.pretty as ct - - def aggregateHashes(executions_json) -> pd.DataFrame: """ Takes the executions, aggregates all the data with sha256 as primary, then returns aggregated dataframe @@ -41,7 +40,6 @@ def aggregateHashes(executions_json) -> pd.DataFrame: return agg_df - def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame: """ Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash, @@ -103,7 +101,6 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame: return aug_df - def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list): if untrusted_publishers is None: untrusted_publishers = [] @@ -146,9 +143,6 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ return needsreview_df, approved_df, unapproved_df - -import pandas as pd - def explode_and_deduplicate(df): df['sha256'] = df['sha256'].str.split(',') df = df.explode('sha256') @@ -205,60 +199,3 @@ def destinationHashes( # Concatenate results df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True) return df_hashdestination - - - -""" - -def explode_and_deduplicate(df): - df['sha256'] = df['sha256'].str.split(',') - df = df.explode('sha256') - return df.drop_duplicates().reset_index(drop=True) - -def clean_sha256(df, column='sha256'): - df[column] = df[column].astype(str).str.strip("'[]\" ") - return df - -def destinationHashes( - df_approved_paths: pd.DataFrame, - df_approved_hashes: pd.DataFrame, - df_hashes_auto_approved: pd.DataFrame, - df_hashes_manually_approved: pd.DataFrame, -): - # Deduplicate and explode all input DataFrames - df_approved_paths = explode_and_deduplicate(df_approved_paths) - df_approved_hashes = explode_and_deduplicate(df_approved_hashes) - df_hashes_auto_approved = explode_and_deduplicate(df_hashes_auto_approved) - df_hashes_manually_approved = explode_and_deduplicate(df_hashes_manually_approved) - - # Clean sha256 values in all relevant DataFrames - df_approved_hashes = clean_sha256(df_approved_hashes) - df_hashes_auto_approved = clean_sha256(df_hashes_auto_approved) - df_hashes_manually_approved = clean_sha256(df_hashes_manually_approved) - - # Create sets for faster lookup - auto_approved_sha256 = set(df_hashes_auto_approved['sha256'].values) - manually_approved_sha256 = set(df_hashes_manually_approved['sha256'].values) - - # Debug: Print unmatched hashes - unmatched = set(df_approved_hashes['sha256']) - (auto_approved_sha256 | manually_approved_sha256) - print(f"Unmatched hashes: {unmatched}") - - # Process df_approved_paths - df_paths = df_approved_paths.assign( - destination='Path Exclusion' - )[['sha256', 'description', 'destination', 'grouped_directory']] - - # Process df_approved_hashes - df_hashes = df_approved_hashes.copy() - df_hashes['destination'] = df_hashes['sha256'].apply( - lambda x: 'Parent Policy Baseline' if x in auto_approved_sha256 - else ('Child Policy Allowlist' if x in manually_approved_sha256 else None) - ) - df_hashes = df_hashes.dropna(subset=['destination']) - df_hashes = df_hashes.assign(grouped_directory=None)[['sha256', 'description', 'destination', 'grouped_directory']] - - # Concatenate results - df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True) - return df_hashdestination -""" \ No newline at end of file diff --git a/utils/pathfunctions.py b/utils/pathfunctions.py index 470de10..0856788 100644 --- a/utils/pathfunctions.py +++ b/utils/pathfunctions.py @@ -19,9 +19,6 @@ from itertools import chain import ast import re - - - def split_path(path): parts = [] while True: @@ -64,7 +61,6 @@ def local_common_pass(paths, min_parts=3): results[path] = best return results - def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3): dirs_series = df[col].astype(str).apply(os.path.dirname) first_pass = local_common_pass(dirs_series.tolist(), min_parts) @@ -72,7 +68,6 @@ def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]]) return df - def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant): """ Compute longest common paths, group filepaths, write CSV for review. @@ -88,8 +83,6 @@ def export_groups_for_review(df, col, group_col, min_number_in_group, path_lengt return filtered, df - - def mask_from_csv(df, csv_path, filepath_col): """ Reads reviewed CSV of groups, keeps only files in approved groups. @@ -120,7 +113,6 @@ def mask_from_csv(df, csv_path, filepath_col): remainder = df[~df[filepath_col].isin(approved_files)].copy() return remainder - def filter_and_drop(approved, eligiblepaths, min_hashes): """ Filters eligiblepaths to rows where all hashes are in approved, @@ -136,7 +128,6 @@ def filter_and_drop(approved, eligiblepaths, min_hashes): return filtered - def inspect_parquet(path): try: df = pd.read_parquet(path) @@ -149,7 +140,6 @@ def inspect_parquet(path): return pd.DataFrame() - def regulator(paths, case_insensitive=True): """ Build a Python raw string regex that matches any of the given Windows path fragments. diff --git a/utils/policyfunctions.py b/utils/policyfunctions.py index aafff35..232a8bc 100644 --- a/utils/policyfunctions.py +++ b/utils/policyfunctions.py @@ -25,9 +25,6 @@ def addHash(policy, hash): def addPath(policy, hash): print(f"Adding the following Path Exclusions to {policy}:") print(hash) - - - def addHashReal(url, allowlistID, hashlist): endpoint = url + '/v1/hash/application/add'