mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Cleanup

Browse Source
This commit is contained in:
=
2025-09-02 08:29:42 -04:00
parent 7eeb072117
commit 6b8c849db8
4 changed files with 1 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/allowlist.py
-3
View File
@@ -16,8 +16,6 @@ import datetime
import requests import requests
import json import json
import os import os
import pandas
import time
import utils.pretty as ct import utils.pretty as ct
import ijson import ijson
import os import os
@@ -158,7 +156,6 @@ def listAllowlists(url):
return choice, policiesnames, policyids return choice, policiesnames, policyids
#Need else and catch for upper bound #Need else and catch for upper bound
def skipback(days): def skipback(days):
""" """
Generate a MongoDB ObjectId for a given number of days ago from today. Generate a MongoDB ObjectId for a given number of days ago from today.
utils/hashfunctions.py
+1 -64
View File
@@ -12,14 +12,13 @@
# #
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
import pandas as pd import pandas as pd
import requests import requests
import os import os
import json import json
import utils.pretty as ct import utils.pretty as ct
def aggregateHashes(executions_json) -> pd.DataFrame: def aggregateHashes(executions_json) -> pd.DataFrame:
""" """
Takes the executions, aggregates all the data with sha256 as primary, then returns aggregated dataframe Takes the executions, aggregates all the data with sha256 as primary, then returns aggregated dataframe
@@ -41,7 +40,6 @@ def aggregateHashes(executions_json) -> pd.DataFrame:
return agg_df return agg_df
def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame: def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
""" """
Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash, Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
@@ -103,7 +101,6 @@ def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
return aug_df return aug_df
def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list): def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
if untrusted_publishers is None: if untrusted_publishers is None:
untrusted_publishers = [] untrusted_publishers = []
@@ -146,9 +143,6 @@ def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publ
return needsreview_df, approved_df, unapproved_df return needsreview_df, approved_df, unapproved_df
import pandas as pd
def explode_and_deduplicate(df): def explode_and_deduplicate(df):
df['sha256'] = df['sha256'].str.split(',') df['sha256'] = df['sha256'].str.split(',')
df = df.explode('sha256') df = df.explode('sha256')
@@ -205,60 +199,3 @@ def destinationHashes(
# Concatenate results # Concatenate results
df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True) df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
return df_hashdestination return df_hashdestination
"""
def explode_and_deduplicate(df):
df['sha256'] = df['sha256'].str.split(',')
df = df.explode('sha256')
return df.drop_duplicates().reset_index(drop=True)
def clean_sha256(df, column='sha256'):
df[column] = df[column].astype(str).str.strip("'[]\" ")
return df
def destinationHashes(
df_approved_paths: pd.DataFrame,
df_approved_hashes: pd.DataFrame,
df_hashes_auto_approved: pd.DataFrame,
df_hashes_manually_approved: pd.DataFrame,
):
# Deduplicate and explode all input DataFrames
df_approved_paths = explode_and_deduplicate(df_approved_paths)
df_approved_hashes = explode_and_deduplicate(df_approved_hashes)
df_hashes_auto_approved = explode_and_deduplicate(df_hashes_auto_approved)
df_hashes_manually_approved = explode_and_deduplicate(df_hashes_manually_approved)
# Clean sha256 values in all relevant DataFrames
df_approved_hashes = clean_sha256(df_approved_hashes)
df_hashes_auto_approved = clean_sha256(df_hashes_auto_approved)
df_hashes_manually_approved = clean_sha256(df_hashes_manually_approved)
# Create sets for faster lookup
auto_approved_sha256 = set(df_hashes_auto_approved['sha256'].values)
manually_approved_sha256 = set(df_hashes_manually_approved['sha256'].values)
# Debug: Print unmatched hashes
unmatched = set(df_approved_hashes['sha256']) - (auto_approved_sha256 | manually_approved_sha256)
print(f"Unmatched hashes: {unmatched}")
# Process df_approved_paths
df_paths = df_approved_paths.assign(
destination='Path Exclusion'
)[['sha256', 'description', 'destination', 'grouped_directory']]
# Process df_approved_hashes
df_hashes = df_approved_hashes.copy()
df_hashes['destination'] = df_hashes['sha256'].apply(
lambda x: 'Parent Policy Baseline' if x in auto_approved_sha256
else ('Child Policy Allowlist' if x in manually_approved_sha256 else None)
)
df_hashes = df_hashes.dropna(subset=['destination'])
df_hashes = df_hashes.assign(grouped_directory=None)[['sha256', 'description', 'destination', 'grouped_directory']]
# Concatenate results
df_hashdestination = pd.concat([df_paths, df_hashes], ignore_index=True)
return df_hashdestination
"""
utils/pathfunctions.py
-10
View File
@@ -19,9 +19,6 @@ from itertools import chain
import ast import ast
import re import re
def split_path(path): def split_path(path):
parts = [] parts = []
while True: while True:
@@ -64,7 +61,6 @@ def local_common_pass(paths, min_parts=3):
results[path] = best results[path] = best
return results return results
def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3): def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min_parts=3):
dirs_series = df[col].astype(str).apply(os.path.dirname) dirs_series = df[col].astype(str).apply(os.path.dirname)
first_pass = local_common_pass(dirs_series.tolist(), min_parts) first_pass = local_common_pass(dirs_series.tolist(), min_parts)
@@ -72,7 +68,6 @@ def add_longest_common_two_local(df, col="filename_x", new_col="longestcfp", min
df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]]) df[new_col] = dirs_series.map(lambda d: second_pass[first_pass[d]])
return df return df
def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant): def export_groups_for_review(df, col, group_col, min_number_in_group, path_length_constant):
""" """
Compute longest common paths, group filepaths, write CSV for review. Compute longest common paths, group filepaths, write CSV for review.
@@ -88,8 +83,6 @@ def export_groups_for_review(df, col, group_col, min_number_in_group, path_lengt
return filtered, df return filtered, df
def mask_from_csv(df, csv_path, filepath_col): def mask_from_csv(df, csv_path, filepath_col):
""" """
Reads reviewed CSV of groups, keeps only files in approved groups. Reads reviewed CSV of groups, keeps only files in approved groups.
@@ -120,7 +113,6 @@ def mask_from_csv(df, csv_path, filepath_col):
remainder = df[~df[filepath_col].isin(approved_files)].copy() remainder = df[~df[filepath_col].isin(approved_files)].copy()
return remainder return remainder
def filter_and_drop(approved, eligiblepaths, min_hashes): def filter_and_drop(approved, eligiblepaths, min_hashes):
""" """
Filters eligiblepaths to rows where all hashes are in approved, Filters eligiblepaths to rows where all hashes are in approved,
@@ -136,7 +128,6 @@ def filter_and_drop(approved, eligiblepaths, min_hashes):
return filtered return filtered
def inspect_parquet(path): def inspect_parquet(path):
try: try:
df = pd.read_parquet(path) df = pd.read_parquet(path)
@@ -149,7 +140,6 @@ def inspect_parquet(path):
return pd.DataFrame() return pd.DataFrame()
def regulator(paths, case_insensitive=True): def regulator(paths, case_insensitive=True):
""" """
Build a Python raw string regex that matches any of the given Windows path fragments. Build a Python raw string regex that matches any of the given Windows path fragments.
utils/policyfunctions.py
-3
View File
@@ -26,9 +26,6 @@ def addPath(policy, hash):
print(f"Adding the following Path Exclusions to {policy}:") print(f"Adding the following Path Exclusions to {policy}:")
print(hash) print(hash)
def addHashReal(url, allowlistID, hashlist): def addHashReal(url, allowlistID, hashlist):
endpoint = url + '/v1/hash/application/add' endpoint = url + '/v1/hash/application/add'
print(ct.colorText("[+] Grabbing All Categories", "cyan")) print(ct.colorText("[+] Grabbing All Categories", "cyan"))