RustImplementation #23

Merged
mysticmomba merged 118 commits from RustImplementation into master 2025-11-04 18:13:24 -05:00
4 changed files with 107 additions and 187 deletions
Showing only changes of commit 908316dc34 - Show all commits
+28 -28
View File
@@ -100,13 +100,13 @@ def sortHashes(
"needs_review": needs_review, "needs_review": needs_review,
"approved": approved, "approved": approved,
"unapproved": unapproved, "unapproved": unapproved,
"unknown" : unknown "leftover" : unknown
} }
for label, records in categories.items(): for label, records in categories.items():
csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{label}_executions.csv" csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_{label}_executions.csv"
html_path = f"{working_dir}\\Needs_Review\\HTML\\{label}.html" html_path = f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{label}.html"
# Convert ExecutionHistoryRecord objects to dictionaries # Convert ExecutionHistoryRecord objects to dictionaries
df = pd.DataFrame([r.__dict__ for r in records]) df = pd.DataFrame([r.__dict__ for r in records])
@@ -125,13 +125,13 @@ def sortHashes(
logger.info(f"Generated HTML report at {html_path}") logger.info(f"Generated HTML report at {html_path}")
def buildPathsandPublishers(split): def buildPathsandPublishers(selected_policies: List[Policy], split):
working_dir = load_env("WORKING_DIR") working_dir = load_env("WORKING_DIR")
df1 = pd.DataFrame() df1 = pd.DataFrame()
df2 = pd.DataFrame() df2 = pd.DataFrame()
all_approved_hashes = pd.DataFrame() all_approved_hashes = pd.DataFrame()
path1 = f"{working_dir}\\Approved\\approved_executions.csv" path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_approved_executions.csv"
path2 = f"{working_dir}\\Approved\\needs_review_executions.csv" path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_needs_review_executions.csv"
if os.path.exists(path1): if os.path.exists(path1):
df1 = pd.read_csv(path1) df1 = pd.read_csv(path1)
@@ -149,10 +149,10 @@ def buildPathsandPublishers(split):
logger.debug(all_approved_hashes.head) logger.debug(all_approved_hashes.head)
else: else:
all_approved_hashes = pd.concat([df1, df2], ignore_index=True) all_approved_hashes = pd.concat([df1, df2], ignore_index=True)
if "filename_exec" in all_approved_hashes.columns: if "filename" in all_approved_hashes.columns:
all_approved_hashes = all_approved_hashes.sort_values(by="filename_exec") all_approved_hashes = all_approved_hashes.sort_values(by="filename")
else: else:
logger.warning("Warning: 'filename_exec' column not found in concatenated DataFrame.") logger.warning("Warning: 'filename' column not found in concatenated DataFrame.")
if not all_approved_hashes.empty: if not all_approved_hashes.empty:
primary_path_exclusions = calculatePath( primary_path_exclusions = calculatePath(
@@ -176,25 +176,25 @@ def buildPathsandPublishers(split):
logger.debug("Preparing to sort dataframes") logger.debug("Preparing to sort dataframes")
for name, df in dataframes.items(): for name, df in dataframes.items():
logger.debug(f" DataFrame headers: {list(df.columns)}") logger.debug(f" DataFrame headers: {list(df.columns)}")
if name == "hashes_to_add": df.sort_values(by="filename_exec", inplace=True) if name == "hashes_to_add": df.sort_values(by="filename", inplace=True)
else: df.sort_values(by="longestcfp", inplace=True) else: df.sort_values(by="longestcfp", inplace=True)
df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{name}.csv", index=False) df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{name}.html") formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{name}.html")
if not all_approved_hashes.empty: if not all_approved_hashes.empty:
# Drop all not signed, only keep unique values # Drop all not signed, only keep unique values
publist = all_approved_hashes[ publist = all_approved_hashes[
all_approved_hashes["publisher_hash"] != "Not Signed" all_approved_hashes["publisher"] != "Not Signed"
].drop_duplicates(subset=["publisher_hash"]) ].drop_duplicates(subset=["publisher"])
# Remove Bad publisher if somehow they made it this far # Remove Bad publisher if somehow they made it this far
pattern = regulator(load_env_json("BAD_PUBLISHERS","[]")) pattern = regulator(load_env_json("BAD_PUBLISHERS","[]"))
publist = publist[~publist["publisher_hash"].str.contains(pattern, na=False)] publist = publist[~publist["publisher"].str.contains(pattern, na=False)]
publist = publist[["publisher_hash"]] publist = publist[["publisher"]]
publist.sort_values(by="publisher_hash", inplace=True) publist.sort_values(by="publisher", inplace=True)
publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\publishers.csv", index=False) publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_publishers.csv", index=False)
def buildPreflights(): def buildPreflights(selected_policies: List[Policy]):
working_dir = load_env("WORKING_DIR") working_dir = load_env("WORKING_DIR")
df1 = pd.DataFrame() df1 = pd.DataFrame()
@@ -202,10 +202,10 @@ def buildPreflights():
approved_hashes = pd.DataFrame() approved_hashes = pd.DataFrame()
approved_publishers = pd.DataFrame() approved_publishers = pd.DataFrame()
hash = f"{working_dir}\\Approved\\hashes_to_add.csv" hash = f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv"
path1 = f"{working_dir}\\Approved\\primary_Paths.csv" path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv"
path2 = f"{working_dir}\\Approved\\secondary_Paths.csv" path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_secondary_Paths.csv"
publishers = f"{working_dir}\\Approved\\publishers.csv" publishers = f"{working_dir}\\Approved\\{selected_policies[0].name}_publishers.csv"
if os.path.exists(hash): if os.path.exists(hash):
approved_hashes = pd.read_csv(hash) approved_hashes = pd.read_csv(hash)
@@ -240,11 +240,11 @@ def buildPreflights():
for name, df in dataframes.items(): for name, df in dataframes.items():
logger.debug(f" DataFrame headers: {list(df.columns)}") logger.debug(f" DataFrame headers: {list(df.columns)}")
if name == "approved_paths":df.sort_values(by="longestcfp", inplace=True) if name == "approved_paths":df.sort_values(by="longestcfp", inplace=True)
elif name == "approved_hashes":df.sort_values(by="filename_exec", inplace=True) elif name == "approved_hashes":df.sort_values(by="filename", inplace=True)
elif name == "approved_publishers" : df.sort_values(by="publisher_hash", inplace=True) elif name == "approved_publishers" : df.sort_values(by="publisher", inplace=True)
df.to_csv(f"{working_dir}\\Preflight\\{name}.csv", index=False) df.to_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{name}.html") formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{selected_policies[0].name}_{name}.html")
def splitFilepathsGrouped(df, col="filename"): def splitFilepathsGrouped(df, col="filename"):
path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int) path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int)
@@ -319,7 +319,7 @@ def calculatePath(approved_hashes, split):
processed_dfs = [] processed_dfs = []
for df in dfs_by_policy: for df in dfs_by_policy:
haslcp = splitFilepathsGrouped(df, "filename_exec") haslcp = splitFilepathsGrouped(df, "filename")
haslcp = haslcp.drop_duplicates() haslcp = haslcp.drop_duplicates()
forbidden = regulator(badpathparts, True) forbidden = regulator(badpathparts, True)
+4 -2
View File
@@ -442,7 +442,9 @@ class ExecutionHistoryRecord:
needs_review = [] needs_review = []
unknown = [] unknown = []
for record in executions: sorted_executions = sorted(executions, key=lambda x: x.filename)
for record in sorted_executions:
decision = getattr(record.hash_obj, "at_decision", None) decision = getattr(record.hash_obj, "at_decision", None)
if decision == "approved": if decision == "approved":
approved.append(record) approved.append(record)
@@ -453,7 +455,7 @@ class ExecutionHistoryRecord:
else: else:
unknown.append(record) unknown.append(record)
logger.info(f"[ExecutionHistoryRecord] Sorted {len(executions)} records by hash_obj.at_decision:") logger.info(f"[ExecutionHistoryRecord] Sorted {len(sorted_executions)} records by hash_obj.at_decision:")
logger.info(f" Approved: {len(approved)}") logger.info(f" Approved: {len(approved)}")
logger.info(f" Unapproved: {len(unapproved)}") logger.info(f" Unapproved: {len(unapproved)}")
logger.info(f" Needs Review: {len(needs_review)}") logger.info(f" Needs Review: {len(needs_review)}")
+15 -15
View File
@@ -133,31 +133,31 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
) )
elif choice == "4": elif choice == "4":
if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"): if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_approved_executions.csv"):
buildPathsandPublishers(False) buildPathsandPublishers(selected_policies, False)
else: else:
print("File not found. Please make sure it's saved correctly and try again.") print("File not found. Please make sure it's saved correctly and try again.")
elif choice == "5": elif choice == "5":
if os.path.exists(f"{working_dir}\\Approved\\hashes_to_add.csv") and os.path.exists( if os.path.exists(f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv") and os.path.exists(
f"{working_dir}\\Approved\\primary_Paths.csv" f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv"
): ):
buildPreflights() buildPreflights(selected_policies)
else: else:
print("File not found. Please make sure it's saved correctly and try again.") print("File not found. Please make sure it's saved correctly and try again.")
elif choice == "6": elif choice == "6":
if ( if (
os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv") os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv")
and os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv") and os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
and destination_policy and destination_policy
and destination_allowlist and destination_allowlist
): ):
print(colorText("These path exclusions would be added to:", "yellow")) print(colorText("These path exclusions would be added to:", "yellow"))
print(destination_policy) print(destination_policy)
pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\approved_paths.csv") pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv")
hashes = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv") hashes = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
unique_combinations = pathexclusions[["longestcfp", "file_extension"]].drop_duplicates() unique_combinations = pathexclusions[["longestcfp", "file_extension"]].drop_duplicates()
@@ -170,14 +170,14 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
print(processed_paths) print(processed_paths)
print(colorText("These publishers would added", "yellow")) print(colorText("These publishers would added", "yellow"))
if os.path.exists(f"{working_dir}\\Preflight\\approved_publishers.csv"): if os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_publishers.csv"):
publishers = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv") publishers = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
if publishers.empty: if publishers.empty:
print(colorText("The publishers list is empty.", "red")) print(colorText("The publishers list is empty.", "red"))
else: else:
processed_publishers = ( processed_publishers = (
publishers[publishers["publisher_hash"] != "Not Signed"] publishers[publishers["publisher"] != "Not Signed"]
["publisher_hash"] ["publisher"]
.drop_duplicates() .drop_duplicates()
.tolist() .tolist()
) )
@@ -194,9 +194,9 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
else: else:
# Log which condition(s) failed # Log which condition(s) failed
missing_items = [] missing_items = []
if not os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv"): if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv"):
missing_items.append("approved_paths.csv not found") missing_items.append("approved_paths.csv not found")
if not os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv"): if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv"):
missing_items.append("approved_hashes.csv not found") missing_items.append("approved_hashes.csv not found")
if not destination_policy: if not destination_policy:
missing_items.append("destination_policy is empty or None") missing_items.append("destination_policy is empty or None")
+59 -141
View File
@@ -187,179 +187,97 @@ def displayIntro():
def printEnforceChecklist(selected_policies, destination_policy, destination_allowlist): def printEnforceChecklist(selected_policies, destination_policy, destination_allowlist):
working_dir = load_env("WORKING_DIR") working_dir = load_env("WORKING_DIR")
print(
colorText(
"\n --------------------------------------------------------------------",
"cyan",
)
)
print(
colorText(
" ------------- 🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒 ------------------",
"cyan",
)
)
print(
colorText(
" --------------------------------------------------------------------",
"cyan",
)
)
print(
colorText(
"\nSequentually follow these steps to prepare a policy for enforcement:",
"white",
)
)
print( def section_header(title):
colorText( print(colorText("\n --------------------------------------------------------------------", "cyan"))
"\n1. Choose which originating policy or policies to move to enforcement", print(colorText(f" ------------- {title} -------------", "cyan"))
"cyan", print(colorText(" --------------------------------------------------------------------", "cyan"))
)
) section_header("🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒")
print(colorText("\nSequentially follow these steps to prepare a policy for enforcement:", "white"))
# Step 1: Originating Policies
print(colorText("\n1. Choose which originating policy or policies to move to enforcement", "cyan"))
if not selected_policies: if not selected_policies:
print(colorText(" [✗] No policies have been chosen", "red")) print(colorText(" [✗] No policies have been chosen", "red"))
else: else:
print(colorText("The following policies have been choosen:", "green")) print(colorText("The following policies have been chosen:", "green"))
for policy in selected_policies: for policy in selected_policies:
print(colorText(f" [✓] {policy.name}", "green")) print(colorText(f" [✓] {policy.name}", "green"))
# Step 2: Destination Policy and Allowlist
print(colorText("2. Choose the destination policy and allowlist", "cyan")) print(colorText("2. Choose the destination policy and allowlist", "cyan"))
if destination_policy:
if not destination_policy:
print(colorText(" [✗] No destination policy has been chosen", "red"))
elif destination_policy:
print(colorText(f" [✓] {destination_policy[0].name} has been selected as the destination policy", "green")) print(colorText(f" [✓] {destination_policy[0].name} has been selected as the destination policy", "green"))
else:
print(colorText(" [✗] No destination policy has been chosen", "red"))
if destination_allowlist:
print(colorText(f" [✓] {destination_allowlist[0].name} has been selected as allowlist", "green"))
if not destination_allowlist: else:
print(colorText(" [✗] No allowlist has been chosen", "red")) print(colorText(" [✗] No allowlist has been chosen", "red"))
elif destination_allowlist:
print(
colorText(
f" [✓] {destination_allowlist[0].name} has been selected as allowlist",
"green",
)
)
# Step 3: Data Preparation
print(colorText("3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", "cyan"))
print( if selected_policies:
colorText( policy_id = selected_policies[0].name
"3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", review_path = f"{working_dir}\\Needs_Review\\Review_First\\{policy_id}_approved_executions.csv"
"cyan", print(colorText(" [✓] Data has been fetched" if os.path.exists(review_path) else " [✗] Data has not been fetched", "green" if os.path.exists(review_path) else "red"))
)
)
if not selected_policies:
print(colorText(" [✗] No policies have been chosen", "red"))
else: else:
if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"): print(colorText(" [✗] No policies selected, cannot check data fetch status", "red"))
print(colorText(" [✓] Data has been fetched", "green"))
else:
print(colorText(" [✗] Data has not been fetched", "red"))
# Step 4: Manual Review
print(colorText("4. Manually review the files:", "cyan")) print(colorText("4. Manually review the files:", "cyan"))
print( if selected_policies:
colorText( policy_id = selected_policies[0].name
" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\unknown_hashes.csv'\n", print(colorText(f" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\{policy_id}_unknown_hashes.csv'\n", "cyan"))
"cyan",
)
)
print(
colorText(
" Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.",
"cyan",
)
)
print(
colorText(
" If metarules need to be created, please make note of them, and remove the row from the csv.",
"cyan",
)
)
print(
colorText(
" When complete, save both csv files to the directory 'approved' and choose this option.",
"cyan",
)
)
print(
colorText(
" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed",
"cyan",
)
)
if os.path.exists(f"{working_dir}\\Approved\\approved_executions.csv"):
print(colorText(" [✓] Reviewed hashes have been loaded", "green"))
else: else:
print(colorText(" [✗] Reviewed hashes have not been loaded", "red")) print(colorText(" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\<policy>_unknown_hashes.csv'\n", "cyan"))
if os.path.exists( print(colorText(" Remove the rows containing hashes you do not approve of...", "cyan"))
f"{working_dir}\\Needs_Review\\Review_Second\\primary_Paths.csv", print(colorText(" When complete, save both csv files to the directory 'approved' and choose this option.", "cyan"))
): print(colorText(" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
print(colorText(" [✓] Path review list created", "green"))
if selected_policies:
policy_id = selected_policies[0].name
approved_path = f"{working_dir}\\Approved\\{policy_id}_approved_executions.csv"
second_review_path = f"{working_dir}\\Needs_Review\\Review_Second\\{policy_id}_primary_Paths.csv"
print(colorText(" [✓] Reviewed hashes have been loaded" if os.path.exists(approved_path) else " [✗] Reviewed hashes have not been loaded", "green" if os.path.exists(approved_path) else "red"))
print(colorText(" [✓] Path review list created" if os.path.exists(second_review_path) else " [✗] Path review list has not been created", "green" if os.path.exists(second_review_path) else "red"))
else: else:
print(colorText(" [✗] Path review list has not been created", "red")) print(colorText(" [✗] No policies selected, cannot check reviewed hashes or path list", "red"))
print( # Step 5: Path Review
colorText( print(colorText("5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'", "cyan"))
"5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'", print(colorText(" Remove the rows containing path exclusions you do not approve of...", "cyan"))
"cyan",
)
)
print(
colorText(
" Remove the rows containing path exclusions you do not approve of. The secondary list can be not added at all if nothing is useful",
"cyan",
)
)
print(colorText(" When complete, save the csv file to the directory 'approved'", "cyan")) print(colorText(" When complete, save the csv file to the directory 'approved'", "cyan"))
print( print(colorText(" Do the same process with the list of publishers for the same directories", "cyan"))
colorText(
" Do the same process with the list of publishers forthe same directories",
"cyan",
)
)
print(colorText(" Preflight Lists will be generated", "cyan")) print(colorText(" Preflight Lists will be generated", "cyan"))
if os.path.exists( if selected_policies:
f"{working_dir}\\Approved\\primary_Paths.csv", policy_id = selected_policies[0].name
): reviewed_path = f"{working_dir}\\Approved\\{policy_id}_primary_Paths.csv"
print(colorText(" [✓] Reviewed path list detected", "green")) preflight_paths = f"{working_dir}\\Preflight\\{policy_id}_approved_paths.csv"
preflight_hashes = f"{working_dir}\\Preflight\\{policy_id}_approved_hashes.csv"
print(colorText(" [✓] Reviewed path list detected" if os.path.exists(reviewed_path) else " [✗] Path review list has not been detected", "green" if os.path.exists(reviewed_path) else "red"))
preflight_ready = os.path.exists(preflight_paths) and os.path.exists(preflight_hashes)
print(colorText(" [✓] Preflight Path Exclusion List has been generated" if preflight_ready else " [✗] Preflight Path Exclusion List has not been generated", "green" if preflight_ready else "red"))
else: else:
print(colorText(" [✗] Path review list has not been detected", "red")) print(colorText(" [✗] No policies selected, cannot check preflight status", "red"))
if os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv") and os.path.exists(
f"{working_dir}\\Preflight\\approved_hashes.csv"
):
print(colorText(" [✓] Preflight Path Exclusion List has been generated", "green"))
else:
print(colorText(" [✗] Preflight Path Exclusion List has not been generated", "red"))
# Final Steps
print(colorText("6. Test ------------------------------------------------------", "cyan")) print(colorText("6. Test ------------------------------------------------------", "cyan"))
print(colorText(" Print rather than apply selected data.", "cyan")) print(colorText(" Print rather than apply selected data.", "cyan"))
print(colorText("7. Liftoff ------------------------------------------------------", "cyan")) print(colorText("7. Liftoff ------------------------------------------------------", "cyan"))
print( print(colorText(" Apply path exclusions according to allowed and approved paths", "cyan"))
colorText(
" Apply path exclusions according to allowed and approved paths",
"cyan",
)
)
print(colorText(" Apply signed or attested hashes to Parent Allow List", "cyan")) print(colorText(" Apply signed or attested hashes to Parent Allow List", "cyan"))
print(colorText(" Apply approved, but unsigned hashes to the Child Allow List", "cyan")) print(colorText(" Apply approved, but unsigned hashes to the Child Allow List", "cyan"))
print( # Utility Options
colorText( print(colorText("R. Remove/Reset Generated data - will prompt to allow keeping execution history", "cyan"))
"R. Remove/Reset Generated data - will prompt to allow keeping execution history",
"cyan",
)
)
print(colorText("F. 📂 - Open Working Directory", "cyan")) print(colorText("F. 📂 - Open Working Directory", "cyan"))
print(colorText("Q. 🔚 - Quit", "cyan")) print(colorText("B. 🔚 - Back", "cyan"))
def areYouSure(): def areYouSure():