Paths fixed... need to double check the path exclusion format to feed api
This commit is contained in:
=
+66
-109
@@ -18,6 +18,7 @@ import gc
|
||||
import json
|
||||
import os
|
||||
import pandas as pd
|
||||
import re
|
||||
import urllib3
|
||||
import utils.allowlist
|
||||
import utils.getdeviceevents
|
||||
@@ -26,13 +27,14 @@ import utils.pathfunctions
|
||||
import utils.policyfunctions
|
||||
import utils.pretty as ct
|
||||
|
||||
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
#Constants
|
||||
url = os.getenv('url')
|
||||
badpublisherlist = ["Brave Software, Inc.", "Zoom Video Communications, Inc.", "GlavSoft LLC"]
|
||||
bad_publisher_list = ["Brave","Zoom", "GlavSoft", "VNC"]
|
||||
pups = ["logmein", "invalid"]
|
||||
badpathparts = ["users", "wwwroot", "windows\\temp", "windows\\task", "windows\\system32", "startup", "windows\\fonts", "Recycle.Bin", "AppData", "programdata"]
|
||||
path_exclusion_constant = 3
|
||||
@@ -277,7 +279,7 @@ def menu_prepare_to_enforce():
|
||||
categorized = utils.hashfunctions.categorizeHashes(
|
||||
pd.read_parquet(f"parquet\\combined_hashlist_{first_policy}_{second_policy}.parquet"),
|
||||
threat_tolerance_constant,
|
||||
badpublisherlist,
|
||||
bad_publisher_list,
|
||||
pups
|
||||
)
|
||||
|
||||
@@ -325,86 +327,15 @@ def menu_prepare_to_enforce():
|
||||
del condensed_combo
|
||||
gc.collect()
|
||||
|
||||
if not os.path.exists(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv"):
|
||||
|
||||
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
|
||||
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet")
|
||||
|
||||
#Pull hash info for the entries in the needs approval table
|
||||
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
|
||||
|
||||
#Deduplicate lists in the columns
|
||||
for col in needsapproval.columns:
|
||||
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
|
||||
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
|
||||
|
||||
#Rename Publisher, Keep and reorder columns we want
|
||||
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
|
||||
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
|
||||
|
||||
needsapproval['filename_key'] = needsapproval['filename'].apply(lambda x: x[0] if isinstance(x, list) and x else '')
|
||||
needsapproval = needsapproval.sort_values(by='filename_key').drop(columns=['filename_key'])
|
||||
|
||||
needsapproval.to_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet",index=False)
|
||||
|
||||
del needsapproval
|
||||
del condensed_combo
|
||||
gc.collect()
|
||||
|
||||
if not os.path.exists(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
|
||||
|
||||
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
|
||||
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet")
|
||||
|
||||
#Pull hash info for the entries in the needs approval table
|
||||
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
|
||||
|
||||
#Deduplicate lists in the columns
|
||||
for col in needsapproval.columns:
|
||||
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
|
||||
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
|
||||
|
||||
#Rename Publisher, Keep and reorder columns we want
|
||||
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
|
||||
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
|
||||
|
||||
needsapproval['filename_key'] = needsapproval['filename'].apply(lambda x: x[0] if isinstance(x, list) and x else '')
|
||||
needsapproval = needsapproval.sort_values(by='filename_key').drop(columns=['filename_key'])
|
||||
|
||||
needsapproval.to_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", index=False)
|
||||
|
||||
del needsapproval
|
||||
del condensed_combo
|
||||
gc.collect()
|
||||
|
||||
if not os.path.exists(f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html"):
|
||||
|
||||
condensed_combo = pd.read_parquet(f"parquet\\condensed_executions_{first_policy}_{second_policy}.parquet")
|
||||
needsapproval= pd.read_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet")
|
||||
|
||||
#Pull hash info for the entries in the needs approval table
|
||||
needsapproval = pd.merge(condensed_combo, needsapproval, on='sha256', how='inner')
|
||||
|
||||
#Deduplicate lists in the columns
|
||||
for col in needsapproval.columns:
|
||||
if needsapproval[col].apply(lambda x: isinstance(x, list)).all():
|
||||
needsapproval[col] = needsapproval[col].apply(deduplicate_list)
|
||||
|
||||
#Rename Publisher, Keep and reorder columns we want
|
||||
needsapproval = needsapproval.rename(columns={'publisher_x': 'publisher'})
|
||||
needsapproval = needsapproval[['sha256', 'publisher', 'description', 'filename', 'hostname', 'username', 'productname', 'productversion','reputation_lastseen', 'reputation_scannermatch', 'reputation_scannercount','reputation_status', 'reputation_threatlevel', 'reputation_threatname','reputation_timestamp', 'pprocess', 'gprocess', 'commandline']]
|
||||
|
||||
needsapproval['filename_key'] = needsapproval['filename'].apply(lambda x: x[0] if isinstance(x, list) and x else '')
|
||||
needsapproval = needsapproval.sort_values(by='filename_key').drop(columns=['filename_key'])
|
||||
|
||||
needsapproval.to_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", index=False)
|
||||
|
||||
del needsapproval
|
||||
del condensed_combo
|
||||
gc.collect()
|
||||
|
||||
|
||||
|
||||
if os.path.exists(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet") & os.path.exists(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet") & os.path.exists(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet"):
|
||||
|
||||
|
||||
utils.hashfunctions.combineHashAndHist(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet", first_policy, second_policy)
|
||||
utils.hashfunctions.combineHashAndHist(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet", first_policy, second_policy)
|
||||
utils.hashfunctions.combineHashAndHist(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet", first_policy, second_policy)
|
||||
|
||||
unknown = pd.read_parquet(f"parquet\\hashes_rep_unknown_{first_policy}_{second_policy}.parquet")
|
||||
good = pd.read_parquet(f"parquet\\hashes_rep_good_{first_policy}_{second_policy}.parquet")
|
||||
bad = pd.read_parquet(f"parquet\\hashes_rep_bad_{first_policy}_{second_policy}.parquet")
|
||||
@@ -423,15 +354,14 @@ def menu_prepare_to_enforce():
|
||||
unknown = unknown[~unknown["filename"].str.contains(pattern, na=False)]
|
||||
good = good[~good["filename"].str.contains(pattern, na=False)]
|
||||
|
||||
|
||||
unknown.to_csv(f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv",index=False)
|
||||
good.to_csv(f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.csv",index=False)
|
||||
bad.to_csv(f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.csv",index=False)
|
||||
|
||||
ct.style_dataframe_dark(unknown, f"needs_approved\\hashes_rep_unknown_{first_policy}_{second_policy}.html")
|
||||
ct.style_dataframe_dark(good, f"needs_approved\\hashes_rep_good_{first_policy}_{second_policy}.html")
|
||||
ct.style_dataframe_dark(bad, f"needs_approved\\hashes_rep_bad_{first_policy}_{second_policy}.html")
|
||||
|
||||
|
||||
elif choice == "3":
|
||||
|
||||
if os.path.exists(f"approved\\hashes_rep_unknown_{first_policy}_{second_policy}.csv") and os.path.exists(f"approved\\hashes_rep_good_{first_policy}_{second_policy}.csv"):
|
||||
@@ -454,54 +384,67 @@ def menu_prepare_to_enforce():
|
||||
if not os.path.exists(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet"):
|
||||
all_approved_hashes = pd.read_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet")
|
||||
print(ct.colorText(f"Beginning calculating longest common filepaths for path exceptions","green"))
|
||||
grouped_df_view, df_with_groups_appended = utils.pathfunctions.export_groups_for_review(all_approved_hashes,"filename","longestcfp",min_files_for_path,path_exclusion_constant)
|
||||
|
||||
df_with_groups_appended.to_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet", index=False)
|
||||
|
||||
|
||||
haslcp = utils.pathfunctions.split_filepaths_grouped(all_approved_hashes)
|
||||
haslcp.drop_duplicates()
|
||||
|
||||
forbidden = utils.pathfunctions.regulator(badpathparts, True)
|
||||
forbidden_lcfp = grouped_df_view["longestcfp"].str.contains(forbidden, na=False)
|
||||
|
||||
# Make a real DataFrame copy before modifying
|
||||
grouped_df_view = grouped_df_view[~forbidden_lcfp].copy()
|
||||
forbidden_lcfp = haslcp["longestcfp"].str.contains(forbidden, na=False)
|
||||
|
||||
|
||||
print(ct.colorText("Removing forbidden filepaths for path exceptions", "green"))
|
||||
|
||||
for col in grouped_df_view.columns:
|
||||
if grouped_df_view[col].apply(lambda x: isinstance(x, list)).all():
|
||||
grouped_df_view[col] = grouped_df_view[col].apply(deduplicate_list)
|
||||
# Make a real DataFrame copy before modifying
|
||||
lcp_not_forbidden = haslcp[~forbidden_lcfp].copy()
|
||||
|
||||
|
||||
grouped_df_view.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet", index=False)
|
||||
grouped_df_view.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv", index=False)
|
||||
ct.style_dataframe_dark(grouped_df_view, f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.html")
|
||||
#For the review, drop down to only the columns we care, and then group by the commmon file path, consolidating and dropping dupes
|
||||
lcp_not_forbidden_review = lcp_not_forbidden[['longestcfp', 'middle', 'filename_only', 'sha256']]
|
||||
|
||||
# Count unique sha256 per longestcfp
|
||||
unique_sha_counts = lcp_not_forbidden_review.groupby('longestcfp')['sha256'].nunique().reset_index()
|
||||
unique_sha_counts.columns = ['longestcfp', 'unique_sha256_count']
|
||||
|
||||
# Merge the count back into the original DataFrame
|
||||
lcp_not_forbidden_review = lcp_not_forbidden_review.merge(unique_sha_counts, on='longestcfp', how='left')
|
||||
lcp_not_forbidden_review = lcp_not_forbidden_review[lcp_not_forbidden_review['unique_sha256_count'] >= min_files_for_path]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
del grouped_df_view
|
||||
del df_with_groups_appended
|
||||
gc.collect()
|
||||
|
||||
lcp_not_forbidden_review.to_parquet(f"parquet\\path_needs_approved_{first_policy}_{second_policy}.parquet",index=False)
|
||||
lcp_not_forbidden_review.to_csv(f"needs_approved\\path_needs_approved_{first_policy}_{second_policy}.csv",index=False)
|
||||
|
||||
|
||||
else:
|
||||
print(ct.colorText(f"Please manually approve hashes prior to this step","red"))
|
||||
|
||||
|
||||
|
||||
elif choice == "4":
|
||||
|
||||
if os.path.exists(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet") and os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
|
||||
if os.path.exists(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv"):
|
||||
if not os.path.exists(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet") and not os.path.exists(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet"):
|
||||
df1 = pd.read_parquet(f"parquet\\approved_hashes_with_paths_{first_policy}_{second_policy}.parquet")
|
||||
allowbyhash = utils.pathfunctions.mask_from_csv(df1, f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv","longestcfp")
|
||||
allhashes = pd.read_parquet(f"parquet\\all_approved_hashes_{first_policy}_{second_policy}.parquet")
|
||||
|
||||
pathexclusions = tryToReadCSV(f"approved\\path_needs_approved_{first_policy}_{second_policy}.csv")
|
||||
|
||||
pathexclusions.to_parquet(f"parquet\\final_path_exclusions_{first_policy}_{second_policy}.parquet", index=False)
|
||||
|
||||
allowbyhash = allhashes[~allhashes['sha256'].isin(pathexclusions['sha256'])]
|
||||
|
||||
allowbyhash.to_parquet(f"parquet\\final_hash_approvals_{first_policy}_{second_policy}.parquet", index=False)
|
||||
|
||||
|
||||
easyview = allowbyhash.groupby('sha256').agg(list).reset_index()
|
||||
# Deduplicate all list columns in easyview
|
||||
for col in easyview.columns:
|
||||
if col != 'sha256': # Skip the grouping column
|
||||
easyview[col] = easyview[col].apply(lambda x: list(set(x)))
|
||||
|
||||
easyview = easyview.sort_values(by=["reputation_status", "filename"])
|
||||
|
||||
ct.style_dataframe_dark(easyview, f"preflight\\final_hash_approvals_{first_policy}_{second_policy}.html")
|
||||
ct.style_dataframe_dark(pathexclusions, f"preflight\\final_path_exclusions_{first_policy}_{second_policy}.html")
|
||||
|
||||
del df1
|
||||
|
||||
del allowbyhash
|
||||
del pathexclusions
|
||||
del easyview
|
||||
@@ -542,7 +485,19 @@ def menu_prepare_to_enforce():
|
||||
print(ct.colorText("Proceeding with the code...", "yellow"))
|
||||
print(ct.colorText(f"Adding path exclusions to {destination_name}", "yellow"))
|
||||
pathexcludelist = pathexclusions['longestcfp'].unique().tolist()
|
||||
utils.policyfunctions.addPath(destination_id,pathexcludelist)
|
||||
|
||||
|
||||
|
||||
# Regex to match a Windows drive letter at the start (e.g., C:\)
|
||||
drive_letter_pattern = re.compile(r'^[a-zA-Z]:\\')
|
||||
|
||||
# Processed list
|
||||
processed_paths = [
|
||||
(path if drive_letter_pattern.match(path) else f"\\\\{path}") + "**"
|
||||
for path in pathexcludelist
|
||||
]
|
||||
|
||||
utils.policyfunctions.addPath(destination_id,processed_paths)
|
||||
|
||||
print(ct.colorText(f"Adding hashes to {allowlist_parent_name}", "yellow"))
|
||||
|
||||
@@ -554,6 +509,8 @@ def menu_prepare_to_enforce():
|
||||
utils.policyfunctions.addHash(allowlist_child_id, allowlist_childhashlist)
|
||||
|
||||
ct.locked()
|
||||
print(repr(processed_paths))
|
||||
print(processed_paths)
|
||||
exit()
|
||||
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user