LocalApproval in progress
This commit is contained in:
+156
-170
@@ -24,6 +24,7 @@ from utils.perstscheduler import register_function, run_once_job, recurring_job,
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
#3rd Party Imports:
|
||||
@@ -49,126 +50,98 @@ def getLocalApprovals(url):
|
||||
os.remove("Local_Approval\\PARQ\\newest_local_approval.parquet")
|
||||
|
||||
#Only keep rows presumably created by the generate local approval function
|
||||
local_approval = local_approval[local_approval['column_name'].str.startswith('🎫 Local Approval 🎫')]
|
||||
local_approval.to_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet", index=False)
|
||||
local_approval = local_approval[local_approval['purpose'].str.startswith('🎫 Local Approval 🎫')]
|
||||
|
||||
local_approval['batchid'] = local_approval['purpose'].apply(lambda x: (match := re.search(r"batch:(\S+)", str(x))) and match.group(1))
|
||||
|
||||
if not local_approval.empty:
|
||||
ct.style_dataframe_dark(local_approval, f"Local_Approval\\HTML\\newest_local_approval.html")
|
||||
local_approval.to_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet", index=False)
|
||||
|
||||
return local_approval
|
||||
|
||||
def getLocalApprovalActivities(url, otpid):
|
||||
pass
|
||||
def getLocalApprovalActivities(url, policy_relationship_map):
|
||||
|
||||
"""
|
||||
def monitorOTP(url, pups):
|
||||
localapprovals = getLocalApprovals(url)
|
||||
|
||||
getLocalApproval(url)
|
||||
newly_added, same_policy, moved_to_audit, moved_to_enforcement, unusual_move = monitorAuditStatus(url, policy_relationship_map)
|
||||
|
||||
register_function("add_hash_and_return_enforcement", returnFromLocalApproval)
|
||||
|
||||
|
||||
|
||||
old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
|
||||
new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
|
||||
|
||||
if os.path.exists(old_otp_path):
|
||||
old_active_OTP = pd.read_parquet(old_otp_path)
|
||||
else:
|
||||
old_active_OTP = pd.DataFrame(columns=['otpid']) # Ensure expected column exists
|
||||
|
||||
current_active_OTP = pd.read_parquet(new_otp_path)
|
||||
|
||||
|
||||
if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
|
||||
if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
|
||||
|
||||
newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
|
||||
still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
|
||||
no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
|
||||
|
||||
|
||||
register_function("addhash", addOTPHashes)
|
||||
|
||||
for _, row in newly_added.iterrows():
|
||||
clientid = row['clientid']
|
||||
duration = (int(row['duration']) * 60)
|
||||
hostname = row['hostname']
|
||||
purpose = row ['purpose']
|
||||
pid = row['otpid']
|
||||
early = math.floor(duration * .95)
|
||||
#If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
|
||||
run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
|
||||
print(f"Processing: {pid} with other data: {row}")
|
||||
|
||||
|
||||
|
||||
#run_once_job(f" ", "addhash", time.time() + early, [url, clientid, pid, pups], None)
|
||||
|
||||
def returnFromLocalApproval(url, device_df, policy_relationship_map, bad_publisher_list, pups, threat_tolerance_constant):
|
||||
# Get unique policy names from device list
|
||||
policies_in_devicelist = sorted(device_df['policy_name'].unique().tolist())
|
||||
|
||||
for _, row in still_in_OTP.iterrows():
|
||||
pid = row['otpid']
|
||||
#While still in OTP, continue to update activities list
|
||||
getOTPActivities(url,pid)
|
||||
# Create inverse map to go from Audit to Enforcement
|
||||
inverse_map = {v: k for k, v in policy_relationship_map.items()}
|
||||
|
||||
for _, row in no_longer_OTP.iterrows():
|
||||
clientid = row['clientid']
|
||||
hostname = row['hostname']
|
||||
purpose = row ['purpose']
|
||||
pid = row['otpid']
|
||||
allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
|
||||
policy = clientf.getPolicyFromClientID(url,clientid)
|
||||
|
||||
"""
|
||||
#Devices can come out of OTP either by timeout, or by early move out of OTP. If they are manually moved out prior to the job to add hashes can run, we want to accelerate the job.
|
||||
# But first, we want to update the OTP activities one final time for the pid, then move up any jobs if they exist, then add the hashes to the local approval allowlist
|
||||
"""
|
||||
|
||||
getOTPActivities(url,pid)
|
||||
find_and_prioritize_jobs_by_pid(pid, 1)
|
||||
addOTPHashes(url, clientid,pid, pups)
|
||||
|
||||
finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
|
||||
finalhashesadded['policy'] = policy
|
||||
finalhashesadded['allowlist'] = allowlist
|
||||
finalhashesadded['added_at'] = time.localtime()
|
||||
|
||||
if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"):
|
||||
df = pd.DataFrame()
|
||||
df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
|
||||
history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
history = pd.concat([history, finalhashesadded], ignore_index=True)
|
||||
ct.style_dataframe_dark(history, f"localapproval_history.html")
|
||||
# Fetch all policies
|
||||
all_policies = policyf.getPolicyDataframe(url)
|
||||
|
||||
history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
|
||||
os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
|
||||
del finalhashesadded
|
||||
# Define policy types
|
||||
policy_types = [1, 2, 6, 7]
|
||||
|
||||
def addOTPHashes(url, clientid, otpid, pups):
|
||||
path = f"OTP\\PARQ\\otp_activities_{otpid}.parquet"
|
||||
activities = pd.read_parquet(path)
|
||||
# Define output directories
|
||||
parq_base_dir = "Local_Approval\\parquet\\"
|
||||
needappr_base_dir = "Local_Approval\\needs_approved"
|
||||
|
||||
pattern = pathf.regulator(pups)
|
||||
allowlist = clientf.getDestAllowlistFromClientID(url, clientid)
|
||||
# Ensure output directories exist
|
||||
os.makedirs(parq_base_dir, exist_ok=True)
|
||||
os.makedirs(needappr_base_dir, exist_ok=True)
|
||||
|
||||
# Initialize or preserve 'hash_added' column
|
||||
if "hash_added" not in activities.columns: activities["hash_added"] = None
|
||||
|
||||
# Identify rows that should be added (not matching pattern and not already added)
|
||||
approve_by_hash = activities[
|
||||
~activities["filename"].str.contains(pattern, na=False) & (activities["hash_added"] != "added")
|
||||
]
|
||||
|
||||
hashes_to_add = approve_by_hash["sha256"].tolist()
|
||||
|
||||
# Add hashes to policy
|
||||
if hashes_to_add:
|
||||
policyf.addHash(url, allowlist, hashes_to_add)
|
||||
|
||||
# Update 'hash_added' column
|
||||
activities["hash_added"] = activities.apply(
|
||||
lambda row: "do not add" if pd.notna(row["filename"]) and pattern in row["filename"]
|
||||
else ("added" if row["sha256"] in hashes_to_add else row["hash_added"]),
|
||||
axis=1
|
||||
# Build execution history
|
||||
policyf.buildExecHistory(
|
||||
url,
|
||||
policies_in_devicelist,
|
||||
parq_base_dir,
|
||||
needappr_base_dir,
|
||||
policy_types,
|
||||
threat_tolerance_constant,
|
||||
bad_publisher_list,
|
||||
pups,
|
||||
)
|
||||
|
||||
# Save the updated DataFrame
|
||||
activities.to_parquet(path)
|
||||
"""
|
||||
# Read hash data
|
||||
unknown_hashes = ct.tryToReadCSV(f"{needappr_base_dir}unknown_hashes.csv")
|
||||
good_hashes = ct.tryToReadCSV(f"{needappr_base_dir}good_hashes.csv")
|
||||
hashes = pd.concat([unknown_hashes, good_hashes], ignore_index=True)
|
||||
|
||||
def generateLocalApprovals(url):
|
||||
# Process each policy
|
||||
for policy in policies_in_devicelist:
|
||||
# Filter for matching policy
|
||||
matching_rows = all_policies[all_policies['name'] == policy]
|
||||
|
||||
if matching_rows.empty:
|
||||
print(f"Warning: No group ID found for policy '{policy}'. Skipping.")
|
||||
continue
|
||||
|
||||
# Extract group ID
|
||||
policy_id = matching_rows['groupid'].values[0]
|
||||
|
||||
# Map to destination ID
|
||||
destination_id = inverse_map.get(policy_id)
|
||||
if destination_id is None:
|
||||
print(f"Warning: No corresponding enforcement policy found for group ID '{policy_id}'. Skipping.")
|
||||
continue
|
||||
|
||||
allowlist = policyf.getDestAllowlist(url, destination_id)
|
||||
|
||||
policyf.addHash(url, allowlist, hashes[hashes['group'] == policy_id])
|
||||
devices_to_move = device_df[device_df['groupid'] == policy_id]
|
||||
devices = devices_to_move['agentid'].drop_duplicates().tolist()
|
||||
for device in devices:
|
||||
clientf.moveAgentToEnforcement(url, device, policy_relationship_map)
|
||||
|
||||
def moveToLocalApproval(url, policy_relationship_map):
|
||||
|
||||
possible_durations = [15, 60, 360, 1440, 10080]
|
||||
duration_selected = None
|
||||
@@ -183,10 +156,10 @@ def generateLocalApprovals(url):
|
||||
duration_selected = possible_durations[choice - 1]
|
||||
print(ct.colorText(f"You selected: {duration_selected}", "yellow"))
|
||||
else:
|
||||
print(ct.colorText("Invalid choice.", "red"))
|
||||
print(ct.colorText("❌ Invalid choice.", "red"))
|
||||
return
|
||||
except ValueError:
|
||||
print(ct.colorText("Invalid input. Please enter a number.", "red"))
|
||||
print(ct.colorText("❌ Invalid input. Please enter a number.", "red"))
|
||||
return
|
||||
|
||||
devicelist = clientf.promptForDevices()
|
||||
@@ -194,14 +167,19 @@ def generateLocalApprovals(url):
|
||||
|
||||
batch = int(time.time())
|
||||
|
||||
if device_df is None or device_df.empty:
|
||||
print(ct.colorText("❌ No agents found or error retrieving agents.", "red"))
|
||||
return
|
||||
|
||||
for row in device_df.itertuples(index=False):
|
||||
try:
|
||||
localapproval(url, batch, duration_selected, row.agentid)
|
||||
addLocalApproval(url, batch, duration_selected, row.agentid)
|
||||
clientf.moveAgentToAudit(url, row.agentid, policy_relationship_map)
|
||||
except Exception as e:
|
||||
print(ct.colorText(f"❌ Error processing agent {row.agentid}: {e}", "red"))
|
||||
|
||||
def localapproval(url, batchid, duration_selected, agentid):
|
||||
purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - starting at {batchid} - for {agentid}"
|
||||
def addLocalApproval(url, batchid, duration_selected, agentid):
|
||||
purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - batch:{batchid} Client:{agentid}"
|
||||
endpoint = url + '/v1/otp/retrieve'
|
||||
payload = {
|
||||
"duration": str(duration_selected),
|
||||
@@ -219,78 +197,86 @@ def localapproval(url, batchid, duration_selected, agentid):
|
||||
except Exception as e:
|
||||
print(ct.colorText(f"An unexpected error occurred for agent {agentid}: {str(e)}", "red"))
|
||||
|
||||
"""
|
||||
def monitorOTP(url, pups):
|
||||
|
||||
getActiveOTP(url)
|
||||
def monitorAuditStatus(url: str, policy_relationship_map: dict):
|
||||
|
||||
# Simulated current agent list
|
||||
current_agent_list = clientf.findAllAgents(url)
|
||||
|
||||
old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
|
||||
new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
|
||||
|
||||
if os.path.exists(old_otp_path):
|
||||
old_active_OTP = pd.read_parquet(old_otp_path)
|
||||
# Load old agent list
|
||||
old_agent_path = "Local_Approval\\PARQ\\last_agent_list.parquet"
|
||||
if os.path.exists(old_agent_path):
|
||||
old_agent_list = pd.read_parquet(old_agent_path)
|
||||
else:
|
||||
old_active_OTP = pd.DataFrame(columns=['otpid']) # Ensure expected column exists
|
||||
old_agent_list = pd.DataFrame(columns=current_agent_list.columns)
|
||||
|
||||
current_active_OTP = pd.read_parquet(new_otp_path)
|
||||
# Merge on hostname
|
||||
merged = pd.merge(
|
||||
old_agent_list[['hostname', 'groupid']],
|
||||
current_agent_list[['hostname', 'groupid']],
|
||||
on='hostname',
|
||||
how='outer',
|
||||
suffixes=('_old', '_current'),
|
||||
indicator=True
|
||||
)
|
||||
|
||||
|
||||
if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
|
||||
if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
|
||||
# Reverse map for enforcement
|
||||
reverse_policy_map = {v: k for k, v in policy_relationship_map.items()}
|
||||
known_transitions = set(policy_relationship_map.items()) | set(reverse_policy_map.items())
|
||||
|
||||
newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
|
||||
still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
|
||||
no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
|
||||
|
||||
# 1. Newly added
|
||||
newly_added = merged[merged['_merge'] == 'right_only']
|
||||
|
||||
register_function("addhash", addOTPHashes)
|
||||
# 2. Same policy
|
||||
same_policy = merged[
|
||||
(merged['_merge'] == 'both') &
|
||||
(merged['groupid_old'] == merged['groupid_current'])
|
||||
]
|
||||
|
||||
for _, row in newly_added.iterrows():
|
||||
clientid = row['clientid']
|
||||
duration = (int(row['duration']) * 60)
|
||||
hostname = row['hostname']
|
||||
purpose = row ['purpose']
|
||||
pid = row['otpid']
|
||||
early = math.floor(duration * .95)
|
||||
#If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
|
||||
run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
|
||||
print(f"Processing: {pid} with other data: {row}")
|
||||
|
||||
# 3. Moved to audit
|
||||
moved_to_audit = merged[
|
||||
(merged['_merge'] == 'both') &
|
||||
(merged['groupid_old'].isin(policy_relationship_map)) &
|
||||
(merged['groupid_current'] == merged['groupid_old'].map(policy_relationship_map))
|
||||
]
|
||||
|
||||
for _, row in still_in_OTP.iterrows():
|
||||
pid = row['otpid']
|
||||
#While still in OTP, continue to update activities list
|
||||
getOTPActivities(url,pid)
|
||||
# 4. Moved to enforcement
|
||||
moved_to_enforcement = merged[
|
||||
(merged['_merge'] == 'both') &
|
||||
(merged['groupid_old'].isin(reverse_policy_map)) &
|
||||
(merged['groupid_current'] == merged['groupid_old'].map(reverse_policy_map))
|
||||
]
|
||||
|
||||
for _, row in no_longer_OTP.iterrows():
|
||||
clientid = row['clientid']
|
||||
hostname = row['hostname']
|
||||
purpose = row ['purpose']
|
||||
pid = row['otpid']
|
||||
allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
|
||||
policy = clientf.getPolicyFromClientID(url,clientid)
|
||||
|
||||
|
||||
getOTPActivities(url,pid)
|
||||
find_and_prioritize_jobs_by_pid(pid, 1)
|
||||
addOTPHashes(url, clientid,pid, pups)
|
||||
|
||||
finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
|
||||
finalhashesadded['policy'] = policy
|
||||
finalhashesadded['allowlist'] = allowlist
|
||||
finalhashesadded['added_at'] = time.localtime()
|
||||
|
||||
if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"):
|
||||
df = pd.DataFrame()
|
||||
df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
|
||||
history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
history = pd.concat([history, finalhashesadded], ignore_index=True)
|
||||
ct.style_dataframe_dark(history, f"localapproval_history.html")
|
||||
# 5. Unusual moves
|
||||
unusual_move = merged[
|
||||
(merged['_merge'] == 'both') &
|
||||
(merged['groupid_old'] != merged['groupid_current']) &
|
||||
merged.apply(lambda row: (row['groupid_old'], row['groupid_current']) not in known_transitions, axis=1)
|
||||
]
|
||||
|
||||
history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
|
||||
|
||||
os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
|
||||
del finalhashesadded
|
||||
"""
|
||||
current_agent_list.to_parquet("Local_Approval\\PARQ\\last_agent_list.parquet", index=False)
|
||||
|
||||
# Return all five DataFrames
|
||||
return newly_added, same_policy, moved_to_audit, moved_to_enforcement, unusual_move
|
||||
|
||||
def getNewLocalApprovals(url):
|
||||
current_la = getLocalApprovals(url)
|
||||
|
||||
# Load old approval list
|
||||
old_la_path = "Local_Approval\\PARQ\\last_la.parquet"
|
||||
if os.path.exists(old_la_path):
|
||||
old_la = pd.read_parquet(old_la_path)
|
||||
else:
|
||||
old_la = pd.DataFrame(columns=current_la.columns)
|
||||
|
||||
# Create composite keys
|
||||
current_la['key'] = current_la['clientid'].astype(str) + "_" + current_la['granted'].astype(str)
|
||||
old_la['key'] = old_la['clientid'].astype(str) + "_" + old_la['granted'].astype(str)
|
||||
|
||||
# Find new entries
|
||||
new_entries = current_la[~current_la['key'].isin(old_la['key'])]
|
||||
|
||||
# Save current approvals for next run
|
||||
current_la.drop(columns=['key'], inplace=True)
|
||||
current_la.to_parquet(old_la_path, index=False)
|
||||
|
||||
return new_entries
|
||||
|
||||
Reference in New Issue
Block a user