mysticmomba/AirlockTools
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

RustImplementation #23

Merged
mysticmomba merged 118 commits from RustImplementation into master 2025-11-04 18:13:24 -05:00
Conversation 0 Commits 118 Files Changed 41
+107 -187
4 changed files with 107 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 908316dc34 - Show all commits
flows/prepPolicy.py
+28 -28
View File
@@ -100,13 +100,13 @@ def sortHashes(
"needs_review": needs_review,
"approved": approved,
"unapproved": unapproved,
"unknown" : unknown
"leftover" : unknown
}
for label, records in categories.items():
csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{label}_executions.csv"
html_path = f"{working_dir}\\Needs_Review\\HTML\\{label}.html"
csv_path = f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_{label}_executions.csv"
html_path = f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{label}.html"
# Convert ExecutionHistoryRecord objects to dictionaries
df = pd.DataFrame([r.__dict__ for r in records])
@@ -125,13 +125,13 @@ def sortHashes(
logger.info(f"Generated HTML report at {html_path}")
def buildPathsandPublishers(split):
def buildPathsandPublishers(selected_policies: List[Policy], split):
working_dir = load_env("WORKING_DIR")
df1 = pd.DataFrame()
df2 = pd.DataFrame()
all_approved_hashes = pd.DataFrame()
path1 = f"{working_dir}\\Approved\\approved_executions.csv"
path2 = f"{working_dir}\\Approved\\needs_review_executions.csv"
path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_approved_executions.csv"
path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_needs_review_executions.csv"
if os.path.exists(path1):
df1 = pd.read_csv(path1)
@@ -149,10 +149,10 @@ def buildPathsandPublishers(split):
logger.debug(all_approved_hashes.head)
else:
all_approved_hashes = pd.concat([df1, df2], ignore_index=True)
if "filename_exec" in all_approved_hashes.columns:
all_approved_hashes = all_approved_hashes.sort_values(by="filename_exec")
if "filename" in all_approved_hashes.columns:
all_approved_hashes = all_approved_hashes.sort_values(by="filename")
else:
logger.warning("Warning: 'filename_exec' column not found in concatenated DataFrame.")
logger.warning("Warning: 'filename' column not found in concatenated DataFrame.")
if not all_approved_hashes.empty:
primary_path_exclusions = calculatePath(
@@ -176,25 +176,25 @@ def buildPathsandPublishers(split):
logger.debug("Preparing to sort dataframes")
for name, df in dataframes.items():
logger.debug(f" DataFrame headers: {list(df.columns)}")
if name == "hashes_to_add": df.sort_values(by="filename_exec", inplace=True)
if name == "hashes_to_add": df.sort_values(by="filename", inplace=True)
else: df.sort_values(by="longestcfp", inplace=True)
df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{name}.html")
df.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Needs_Review\\HTML\\{selected_policies[0].name}_{name}.html")
if not all_approved_hashes.empty:
# Drop all not signed, only keep unique values
publist = all_approved_hashes[
all_approved_hashes["publisher_hash"] != "Not Signed"
].drop_duplicates(subset=["publisher_hash"])
all_approved_hashes["publisher"] != "Not Signed"
].drop_duplicates(subset=["publisher"])
# Remove Bad publisher if somehow they made it this far
pattern = regulator(load_env_json("BAD_PUBLISHERS","[]"))
publist = publist[~publist["publisher_hash"].str.contains(pattern, na=False)]
publist = publist[["publisher_hash"]]
publist.sort_values(by="publisher_hash", inplace=True)
publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\publishers.csv", index=False)
publist = publist[~publist["publisher"].str.contains(pattern, na=False)]
publist = publist[["publisher"]]
publist.sort_values(by="publisher", inplace=True)
publist.to_csv(f"{working_dir}\\Needs_Review\\Review_Second\\{selected_policies[0].name}_publishers.csv", index=False)
def buildPreflights():
def buildPreflights(selected_policies: List[Policy]):
working_dir = load_env("WORKING_DIR")
df1 = pd.DataFrame()
@@ -202,10 +202,10 @@ def buildPreflights():
approved_hashes = pd.DataFrame()
approved_publishers = pd.DataFrame()
hash = f"{working_dir}\\Approved\\hashes_to_add.csv"
path1 = f"{working_dir}\\Approved\\primary_Paths.csv"
path2 = f"{working_dir}\\Approved\\secondary_Paths.csv"
publishers = f"{working_dir}\\Approved\\publishers.csv"
hash = f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv"
path1 = f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv"
path2 = f"{working_dir}\\Approved\\{selected_policies[0].name}_secondary_Paths.csv"
publishers = f"{working_dir}\\Approved\\{selected_policies[0].name}_publishers.csv"
if os.path.exists(hash):
approved_hashes = pd.read_csv(hash)
@@ -240,11 +240,11 @@ def buildPreflights():
for name, df in dataframes.items():
logger.debug(f" DataFrame headers: {list(df.columns)}")
if name == "approved_paths":df.sort_values(by="longestcfp", inplace=True)
elif name == "approved_hashes":df.sort_values(by="filename_exec", inplace=True)
elif name == "approved_publishers" : df.sort_values(by="publisher_hash", inplace=True)
elif name == "approved_hashes":df.sort_values(by="filename", inplace=True)
elif name == "approved_publishers" : df.sort_values(by="publisher", inplace=True)
df.to_csv(f"{working_dir}\\Preflight\\{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{name}.html")
df.to_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_{name}.csv", index=False)
formatHTML(df, f"{working_dir}\\Preflight\\HTML\\{selected_policies[0].name}_{name}.html")
def splitFilepathsGrouped(df, col="filename"):
path_exclusion_constant = get_protected_value("PATH_EXCLUSION_CONST", cast_type= int)
@@ -319,7 +319,7 @@ def calculatePath(approved_hashes, split):
processed_dfs = []
for df in dfs_by_policy:
haslcp = splitFilepathsGrouped(df, "filename_exec")
haslcp = splitFilepathsGrouped(df, "filename")
haslcp = haslcp.drop_duplicates()
forbidden = regulator(badpathparts, True)
models/execution.py
+4 -2
View File
@@ -442,7 +442,9 @@ class ExecutionHistoryRecord:
needs_review = []
unknown = []
for record in executions:
sorted_executions = sorted(executions, key=lambda x: x.filename)
for record in sorted_executions:
decision = getattr(record.hash_obj, "at_decision", None)
if decision == "approved":
approved.append(record)
@@ -453,7 +455,7 @@ class ExecutionHistoryRecord:
else:
unknown.append(record)
logger.info(f"[ExecutionHistoryRecord] Sorted {len(executions)} records by hash_obj.at_decision:")
logger.info(f"[ExecutionHistoryRecord] Sorted {len(sorted_executions)} records by hash_obj.at_decision:")
logger.info(f" Approved: {len(approved)}")
logger.info(f" Unapproved: {len(unapproved)}")
logger.info(f" Needs Review: {len(needs_review)}")
utils/menus.py
+15 -15
View File
@@ -133,31 +133,31 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
)
elif choice == "4":
if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"):
buildPathsandPublishers(False)
if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\{selected_policies[0].name}_approved_executions.csv"):
buildPathsandPublishers(selected_policies, False)
else:
print("File not found. Please make sure it's saved correctly and try again.")
elif choice == "5":
if os.path.exists(f"{working_dir}\\Approved\\hashes_to_add.csv") and os.path.exists(
f"{working_dir}\\Approved\\primary_Paths.csv"
if os.path.exists(f"{working_dir}\\Approved\\{selected_policies[0].name}_hashes_to_add.csv") and os.path.exists(
f"{working_dir}\\Approved\\{selected_policies[0].name}_primary_Paths.csv"
):
buildPreflights()
buildPreflights(selected_policies)
else:
print("File not found. Please make sure it's saved correctly and try again.")
elif choice == "6":
if (
os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv")
and os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv")
os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv")
and os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
and destination_policy
and destination_allowlist
):
print(colorText("These path exclusions would be added to:", "yellow"))
print(destination_policy)
pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\approved_paths.csv")
hashes = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv")
pathexclusions = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv")
hashes = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
unique_combinations = pathexclusions[["longestcfp", "file_extension"]].drop_duplicates()
@@ -170,14 +170,14 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
print(processed_paths)
print(colorText("These publishers would added", "yellow"))
if os.path.exists(f"{working_dir}\\Preflight\\approved_publishers.csv"):
publishers = pd.read_csv(f"{working_dir}\\Preflight\\approved_hashes.csv")
if os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_publishers.csv"):
publishers = pd.read_csv(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv")
if publishers.empty:
print(colorText("The publishers list is empty.", "red"))
else:
processed_publishers = (
publishers[publishers["publisher_hash"] != "Not Signed"]
["publisher_hash"]
publishers[publishers["publisher"] != "Not Signed"]
["publisher"]
.drop_duplicates()
.tolist()
)
@@ -194,9 +194,9 @@ def menu_policy_enforce(api: AirlockAPIWrapper):
else:
# Log which condition(s) failed
missing_items = []
if not os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv"):
if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_paths.csv"):
missing_items.append("approved_paths.csv not found")
if not os.path.exists(f"{working_dir}\\Preflight\\approved_hashes.csv"):
if not os.path.exists(f"{working_dir}\\Preflight\\{selected_policies[0].name}_approved_hashes.csv"):
missing_items.append("approved_hashes.csv not found")
if not destination_policy:
missing_items.append("destination_policy is empty or None")
utils/utils.py
+60 -142
View File
@@ -187,179 +187,97 @@ def displayIntro():
def printEnforceChecklist(selected_policies, destination_policy, destination_allowlist):
working_dir = load_env("WORKING_DIR")
print(
colorText(
"\n --------------------------------------------------------------------",
"cyan",
)
)
print(
colorText(
" ------------- 🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒 ------------------",
"cyan",
)
)
print(
colorText(
" --------------------------------------------------------------------",
"cyan",
)
)
print(
colorText(
"\nSequentually follow these steps to prepare a policy for enforcement:",
"white",
)
)
print(
colorText(
"\n1. Choose which originating policy or policies to move to enforcement",
"cyan",
)
)
def section_header(title):
print(colorText("\n --------------------------------------------------------------------", "cyan"))
print(colorText(f" ------------- {title} -------------", "cyan"))
print(colorText(" --------------------------------------------------------------------", "cyan"))
section_header("🛠️ 🔒 Prepare to Enforce Policy 🛠️ 🔒")
print(colorText("\nSequentially follow these steps to prepare a policy for enforcement:", "white"))
# Step 1: Originating Policies
print(colorText("\n1. Choose which originating policy or policies to move to enforcement", "cyan"))
if not selected_policies:
print(colorText(" [✗] No policies have been chosen", "red"))
else:
print(colorText("The following policies have been choosen:", "green"))
print(colorText("The following policies have been chosen:", "green"))
for policy in selected_policies:
print(colorText(f" [✓] {policy.name}", "green"))
print(colorText("2. Choose the destination policy and allowlist", "cyan"))
if not destination_policy:
print(colorText(" [✗] No destination policy has been chosen", "red"))
elif destination_policy:
# Step 2: Destination Policy and Allowlist
print(colorText("2. Choose the destination policy and allowlist", "cyan"))
if destination_policy:
print(colorText(f" [✓] {destination_policy[0].name} has been selected as the destination policy", "green"))
else:
print(colorText(" [✗] No destination policy has been chosen", "red"))
if not destination_allowlist:
if destination_allowlist:
print(colorText(f" [✓] {destination_allowlist[0].name} has been selected as allowlist", "green"))
else:
print(colorText(" [✗] No allowlist has been chosen", "red"))
elif destination_allowlist:
print(
colorText(
f" [✓] {destination_allowlist[0].name} has been selected as allowlist",
"green",
)
)
print(
colorText(
"3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes",
"cyan",
)
)
if not selected_policies:
print(colorText(" [✗] No policies have been chosen", "red"))
# Step 3: Data Preparation
print(colorText("3. Pull and stage event history, combine the histories, add hash info, then categorize the hashes", "cyan"))
if selected_policies:
policy_id = selected_policies[0].name
review_path = f"{working_dir}\\Needs_Review\\Review_First\\{policy_id}_approved_executions.csv"
print(colorText(" [✓] Data has been fetched" if os.path.exists(review_path) else " [✗] Data has not been fetched", "green" if os.path.exists(review_path) else "red"))
else:
if os.path.exists(f"{working_dir}\\Needs_Review\\Review_First\\approved_executions.csv"):
print(colorText(" [✓] Data has been fetched", "green"))
else:
print(colorText(" [✗] Data has not been fetched", "red"))
print(colorText(" [✗] No policies selected, cannot check data fetch status", "red"))
# Step 4: Manual Review
print(colorText("4. Manually review the files:", "cyan"))
print(
colorText(
" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\unknown_hashes.csv'\n",
"cyan",
)
)
print(
colorText(
" Remove the rows containing hashes you do not approve of, and those you would not approve of without metarules.",
"cyan",
)
)
print(
colorText(
" If metarules need to be created, please make note of them, and remove the row from the csv.",
"cyan",
)
)
print(
colorText(
" When complete, save both csv files to the directory 'approved' and choose this option.",
"cyan",
)
)
print(
colorText(
" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed",
"cyan",
)
)
if os.path.exists(f"{working_dir}\\Approved\\approved_executions.csv"):
print(colorText(" [✓] Reviewed hashes have been loaded", "green"))
if selected_policies:
policy_id = selected_policies[0].name
print(colorText(f" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\{policy_id}_unknown_hashes.csv'\n", "cyan"))
else:
print(colorText(" [✗] Reviewed hashes have not been loaded", "red"))
print(colorText(" 'prepare_policy\\needs_approved\\good_hashes.csv' and 'prepare_policy\\needs_approved\\<policy>_unknown_hashes.csv'\n", "cyan"))
if os.path.exists(
f"{working_dir}\\Needs_Review\\Review_Second\\primary_Paths.csv",
):
print(colorText(" [✓] Path review list created", "green"))
print(colorText(" Remove the rows containing hashes you do not approve of...", "cyan"))
print(colorText(" When complete, save both csv files to the directory 'approved' and choose this option.", "cyan"))
print(colorText(" This will combine these approved hashes with the automatically approved hashes and generate a list of paths to be reviewed", "cyan"))
if selected_policies:
policy_id = selected_policies[0].name
approved_path = f"{working_dir}\\Approved\\{policy_id}_approved_executions.csv"
second_review_path = f"{working_dir}\\Needs_Review\\Review_Second\\{policy_id}_primary_Paths.csv"
print(colorText(" [✓] Reviewed hashes have been loaded" if os.path.exists(approved_path) else " [✗] Reviewed hashes have not been loaded", "green" if os.path.exists(approved_path) else "red"))
print(colorText(" [✓] Path review list created" if os.path.exists(second_review_path) else " [✗] Path review list has not been created", "green" if os.path.exists(second_review_path) else "red"))
else:
print(colorText(" [✗] Path review list has not been created", "red"))
print(colorText(" [✗] No policies selected, cannot check reviewed hashes or path list", "red"))
print(
colorText(
"5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'",
"cyan",
)
)
print(
colorText(
" Remove the rows containing path exclusions you do not approve of. The secondary list can be not added at all if nothing is useful",
"cyan",
)
)
# Step 5: Path Review
print(colorText("5. Manually review the files \n 'prepare_policy\\needs_approved\\primary_Paths.csv'\n 'prepare_policy\\needs_approved\\secondary_Paths.csv'", "cyan"))
print(colorText(" Remove the rows containing path exclusions you do not approve of...", "cyan"))
print(colorText(" When complete, save the csv file to the directory 'approved'", "cyan"))
print(
colorText(
" Do the same process with the list of publishers forthe same directories",
"cyan",
)
)
print(colorText(" Do the same process with the list of publishers for the same directories", "cyan"))
print(colorText(" Preflight Lists will be generated", "cyan"))
if os.path.exists(
f"{working_dir}\\Approved\\primary_Paths.csv",
):
print(colorText(" [✓] Reviewed path list detected", "green"))
if selected_policies:
policy_id = selected_policies[0].name
reviewed_path = f"{working_dir}\\Approved\\{policy_id}_primary_Paths.csv"
preflight_paths = f"{working_dir}\\Preflight\\{policy_id}_approved_paths.csv"
preflight_hashes = f"{working_dir}\\Preflight\\{policy_id}_approved_hashes.csv"
print(colorText(" [✓] Reviewed path list detected" if os.path.exists(reviewed_path) else " [✗] Path review list has not been detected", "green" if os.path.exists(reviewed_path) else "red"))
preflight_ready = os.path.exists(preflight_paths) and os.path.exists(preflight_hashes)
print(colorText(" [✓] Preflight Path Exclusion List has been generated" if preflight_ready else " [✗] Preflight Path Exclusion List has not been generated", "green" if preflight_ready else "red"))
else:
print(colorText(" [✗] Path review list has not been detected", "red"))
if os.path.exists(f"{working_dir}\\Preflight\\approved_paths.csv") and os.path.exists(
f"{working_dir}\\Preflight\\approved_hashes.csv"
):
print(colorText(" [✓] Preflight Path Exclusion List has been generated", "green"))
else:
print(colorText(" [✗] Preflight Path Exclusion List has not been generated", "red"))
print(colorText(" [✗] No policies selected, cannot check preflight status", "red"))
# Final Steps
print(colorText("6. Test ------------------------------------------------------", "cyan"))
print(colorText(" Print rather than apply selected data.", "cyan"))
print(colorText("7. Liftoff ------------------------------------------------------", "cyan"))
print(
colorText(
" Apply path exclusions according to allowed and approved paths",
"cyan",
)
)
print(colorText(" Apply path exclusions according to allowed and approved paths", "cyan"))
print(colorText(" Apply signed or attested hashes to Parent Allow List", "cyan"))
print(colorText(" Apply approved, but unsigned hashes to the Child Allow List", "cyan"))
print(
colorText(
"R. Remove/Reset Generated data - will prompt to allow keeping execution history",
"cyan",
)
)
# Utility Options
print(colorText("R. Remove/Reset Generated data - will prompt to allow keeping execution history", "cyan"))
print(colorText("F. 📂 - Open Working Directory", "cyan"))
print(colorText("Q. 🔚 - Quit", "cyan"))
print(colorText("B. 🔚 - Back", "cyan"))
def areYouSure():