LocalApproval in progress

AirlockTools.py

@@ -21,6 +21,7 @@ import utils.localapproval as la
 import utils.otpfunctions
 import utils.policyfunctions
 import utils.utils as ct
+import pandas as pd
 
 from utils.perstscheduler import register_function, run_once_job, recurring_job, reload_jobs, start_scheduler
 
@@ -41,7 +42,8 @@ policy_relationship_map ={ #Enforcement : Audit
 "bf0b1f9b-bfea-4f44-97c0-80e27ff61712" : "538d3218-92f4-4943-a6ee-db9267ab62d8", #AT Servers General, AT Servers General Audit
 "31ababac-65de-4c6a-86dd-6691d7e3ee3b" : "fc05b42a-b846-4e72-88ca-c35d416e699f", #AT Epic, #AT Epic Audit
 "d1f58960-f866-49e0-848a-a5b09fffd4cd" : "d55c03a6-c376-4391-8626-4f843b882a7c", #AT DMZ Enforced, #AT DMZ Audit
- "504dd011-86b6-489a-b78f-eff589cef8aa" : "88a1cfdc-3b30-448b-b309-be16fe437ca3" #AT Workstations BCA, #AT Workstations BCA Audit3
+"504dd011-86b6-489a-b78f-eff589cef8aa" : "88a1cfdc-3b30-448b-b309-be16fe437ca3", #AT Workstations BCA, #AT Workstations BCA Audit
+"d126db36-72ed-4937-adc7-d88b7509a5b5" : "5aebf6a0-1d67-47b4-9c5f-2866ffca5671" #AT Testing, AT Testing
 }
  
 def main():
@@ -64,7 +66,7 @@ def main():
                     
             register_function("monitorOTP", utils.otpfunctions.monitorOTP)
             register_function("updateAudit", utils.policyfunctions.updateAuditPoliciesFromEnforcementPolices)
-            register_function("monitorLA", la.monitorLA)
+           # register_function("monitorLA", la.monitorLA)
             
             if not os.path.exists("scheduling\\jobs.json"): 
                 recurring_job("monitorOTP", "monitorOTP", interval=60, unit="seconds", args=[url, pups])
@@ -108,10 +110,16 @@ def menu_main():
             utils.clientfunctions.findAgents(url, devicelist, False)
 
         elif choice == "7":
-            la.generateLocalApprovals(url)
+            pass
+
 
         elif choice == "8":
-            p
+            las = la.getLocalApprovals(url)
+            las.to_csv("la.csv", index=False)
+        elif choice == "10":
+            utils.clientfunctions.moveAgentToAudit(url,"6a221ece-0c10-4eb8-b1e5-06a1000a5696",policy_relationship_map)
+        elif choice == "11":
+            utils.clientfunctions.moveAgentToEnforcement(url,"6a221ece-0c10-4eb8-b1e5-06a1000a5696",policy_relationship_map)
         elif choice == "Q":
             break
         else:

utils/clientfunctions.py

@@ -29,6 +29,8 @@ import re
 import pandas as pd
 import requests
 
+
+
 def findAgentID(url):
        
     print(ct.colorText("WARNING: Device Name is Case Sensitive", "red"))
@@ -65,16 +67,9 @@ def getDestAllowlistFromClientID(url, clientid):
     result = json.loads(response.text)
     data = pd.DataFrame(result["response"]["agents"])
 
-    allowlists = getPolicyAllowlists(url,data.loc[0, "groupid"]) 
+    allowlist = getDestAllowlist(url,data.loc[0, "groupid"]) 
  
-    matches = allowlists.loc[
+    return allowlist
-        allowlists['name'].str.contains('local', case=False, na=False) &
-        allowlists['name'].str.contains('approval', case=False, na=False),
-        'applicationid'
-    ].values
-    app_id = matches[0] if len(matches) > 0 else None
-    
-    return app_id
 
 def getPolicyFromClientID(url, clientid):
 
@@ -91,24 +86,11 @@ def getPolicyFromClientID(url, clientid):
     result = json.loads(response.text)
     data = pd.DataFrame(result["response"]["agents"])
 
-    policy = getPolicyName(url,data.loc[0, "groupid"]) 
+    policy_name = getPolicyName(url,data.loc[0, "groupid"]) 
-  
+    policy_id   = data.loc[0, "groupid"]
-    return policy
+    return policy_name, policy_id
     
-def getPolicyAllowlists(url, groupid):
 
-    endpoint = url + '/v1/group/policies'
-    payload = {
-            "groupid" : f"{groupid}"
-    }
-
-    headers = {"X-APIKey": os.getenv('APIKEY')}
-    payload = json.dumps(payload)
-
-    response = requests.post(endpoint, headers=headers, data=payload, verify=False)
-    result = json.loads(response.text)
-    data = pd.DataFrame(result["response"]["applications"])
-    return data
 
 def getPolicyName(url, groupid):
     endpoint = url + '/v1/group/'
@@ -272,90 +254,9 @@ def promptForDevices():
 
     return device_input_str
 
-
-def returnToEnforcement(url, device_df, policy_relationship_map, bad_publisher_list, pups, badpathparts, threat_tolerance_constant, path_exclusion_constant, min_files_for_path):
-    policylist = sorted(device_df['policy_name'].unique().tolist())
-    type = [1, 2, 6, 7]
-  
-    parq_base_dir = "prepare_policy\\parquet\\"
-    appr_base_dir = "prepare_policy\\approved\\"
-    needappr_base_dir = "prepare_policy\\needs_approved"
-    pflight_base_dir = "prepare_policy\\preflight"
-    
-    #If the directorys where we're going to store our output dont exist, make them. 
-    os.makedirs(parq_base_dir, exist_ok=True)
-    os.makedirs(needappr_base_dir, exist_ok=True)
-    os.makedirs(appr_base_dir, exist_ok=True)
-    os.makedirs(pflight_base_dir, exist_ok=True)
-
-    while True:
-        
-        #ct.printDeviceEnforceChecklist()
-        
-        choice = input(ct.colorText("\nEnter your choice: ", "white"))
-        
-        if choice == "1":
-            
-            policyf.buildExecHistory(url,
-                        policylist, 
-                        parq_base_dir, 
-                        needappr_base_dir, 
-                        type, 
-                        threat_tolerance_constant,
-                        bad_publisher_list,
-                        pups,
-                        )
-            
-        elif choice == "2":
-            
-            pathf.generateApprovables(needappr_base_dir, appr_base_dir, parq_base_dir, badpathparts, bad_publisher_list, path_exclusion_constant, min_files_for_path, True)
-                
-        elif choice == "3":
-            policyf.savePreFlights(appr_base_dir, parq_base_dir, pflight_base_dir)
-                     
-        """
-
-        elif choice == "4":
-            if os.path.exists(f"preflight\\final_path_exclusions.html") and os.path.exists(f"preflight\\final_hash_approvals.html")    and allowlist_parent_name != " " and allowlist_child_name != " " and destination_name != " ":
-                sendToPolicyTest(
-                    url,
-                    first_policy,
-                    second_policy,
-                    destination_name,
-                    destination_id,
-                    allowlist_parent_name,
-                    allowlist_parent_id, 
-                    allowlist_child_name,
-                    allowlist_child_id
-                    )
-
-        elif choice == "5":
-            if os.path.exists(f"preflight\\final_path_exclusions.html") and os.path.exists(f"preflight\\final_hash_approvals.html")    and allowlist_parent_name != " " and allowlist_child_name != " " and destination_name != " ":
-                sendToPolicy(
-                    url,
-                    first_policy,
-                    second_policy,
-                    destination_name,
-                    destination_id,
-                    allowlist_parent_name,
-                    allowlist_parent_id, 
-                    allowlist_child_name,
-                    allowlist_child_id
-                    )
-        elif choice == "R":
-            
-            pathf.clean_folders(enforcement_prep)
-
-        elif choice == "Q":
-            break
-        else:
-            print(ct.colorText("Invalid choice. Please try again.", "red"))    
-    """
-        
-
 def findQuietAgents(url):
     # Get policy selection and agent list
-    choice, policynames, policyid = policyf.listPolicies(url)
+    choice, policynames, policyid = policyf.choosePolicies(url)
     policy = policynames[choice]
     groupid = policyid[choice]
     agents = findGroupAgents(url, groupid)
@@ -418,15 +319,18 @@ def findQuietAgents(url):
     print(ct.colorText(f"Saving CSV to {policy}_agents_last_{history_days}_days.csv", "green"))
     agents.to_csv(f"{policy}_agents_last_{history_days}_days.csv", index=False)
 
-    # Summary stats
-    zero_count = (agents['execution_count'] == 0).sum()
-    total_hosts = len(agents)
-    zero_percentage = (zero_count / total_hosts) * 100
 
-    print(f"Number of hosts in policy: {total_hosts}")
+    # Summary statistics
-    print(f"Number of hosts with execution_count = 0: {zero_count}")
+    total_agents = len(agents)
-    print(f"Percentage of hosts with execution_count = 0: {zero_percentage:.2f}%")
+    ready_agents = agents['enforce_ready'].sum()
+    not_ready_agents = total_agents - ready_agents
+    ready_percentage = (ready_agents / total_agents) * 100
 
+    # Print results
+    print(f"Total agents: {total_agents}")
+    print(f"Agents marked as 'enforce_ready': {ready_agents}")
+    print(f"Agents not ready: {not_ready_agents}")
+    print(f"Percentage ready for enforcement: {ready_percentage:.2f}%")
 
 def findGroupAgents(url, groupid):        
     endpoint = url + '/v1/agent/find'
@@ -456,4 +360,74 @@ def findGroupAgents(url, groupid):
 
     data['status'] = data['status'].map(status_map)
     return(data)
-    return(data)
+
+def moveAgentToAudit(url, agentid, policy_relationship_map):
+    policy_name, policy_id = getPolicyFromClientID(url, agentid)
+
+    print(policy_id)
+
+
+    if policy_id in policy_relationship_map:
+        target_policy = policy_relationship_map[policy_id]
+    elif policy_id in policy_relationship_map.values():
+        print(f"Agent {agentid} is already in an audit group. No action needed.")
+        return
+    else:
+        print(f"Error: No corresponding audit policy found for policy: {policy_name} - {policy_id}.")
+        return
+
+    moveAgent(url, agentid, target_policy, "audit")
+
+def moveAgentToEnforcement(url, agentid, policy_relationship_map):
+    policy_name, policy_id = getPolicyFromClientID(url, agentid)
+
+    # Invert the map for audit → enforcement
+    inverse_map = {v: k for k, v in policy_relationship_map.items()}
+
+    if policy_id in inverse_map:
+        target_policy = inverse_map[policy_id]
+    elif policy_id in inverse_map.values():
+        print(f"Agent {agentid} is already in an enforcement group. No action needed.")
+        return
+    else:
+        print(f"Error: No corresponding enforcement policy found for policy: {policy_name} - {policy_id}.")
+        return
+
+    moveAgent(url, agentid, target_policy, "enforcement")
+
+def moveAgent(url, agentid, target_policy, direction):
+    endpoint = f"{url}/v1/agent/move"
+    payload = {
+        "groupid": target_policy,
+        "agentid": agentid
+    }
+
+    headers = {"X-APIKey": os.getenv('APIKEY')}
+    response = None  # Initialize to avoid unbound errors
+
+    try:
+        response = requests.post(endpoint, headers=headers, data=json.dumps(payload), verify=False)
+        response.raise_for_status()  # Raises HTTPError for bad status codes
+
+        result = response.json()
+
+        # Check if 'error' key exists and if it's not a success message
+        if "error" in result and result["error"].lower() != "success":
+            print(f"API returned an error: {result['error']}")
+        else:
+            print(f"✅ Agent {agentid} successfully moved to {direction} group {target_policy}.")
+
+    except requests.exceptions.HTTPError as http_err:
+        print(f"HTTP error occurred: {http_err}")
+        if response is not None:
+            print("Raw response:", response.text)
+    except requests.exceptions.RequestException as req_err:
+        print(f"Request error occurred: {req_err}")
+    except ValueError:
+        print("Failed to parse JSON response.")
+        if response is not None:
+            print("Raw response:", response.text)
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+        if response is not None:
+            print("Raw response:", response.text)

utils/localapproval.py

@@ -24,6 +24,7 @@ from utils.perstscheduler import register_function, run_once_job, recurring_job,
 import json
 import math
 import os
+import re
 import time
 
 #3rd Party Imports:
@@ -49,126 +50,98 @@ def getLocalApprovals(url):
         os.remove("Local_Approval\\PARQ\\newest_local_approval.parquet")
 
     #Only keep rows presumably created by the generate local approval function
-    local_approval = local_approval[local_approval['column_name'].str.startswith('🎫 Local Approval 🎫')]
+    local_approval = local_approval[local_approval['purpose'].str.startswith('🎫 Local Approval 🎫')]
-    local_approval.to_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet", index=False)
+
+    local_approval['batchid'] = local_approval['purpose'].apply(lambda x: (match := re.search(r"batch:(\S+)", str(x))) and match.group(1))
+    
     if not local_approval.empty:
         ct.style_dataframe_dark(local_approval, f"Local_Approval\\HTML\\newest_local_approval.html")
+        local_approval.to_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet", index=False)
+    
+    return local_approval
    
-def getLocalApprovalActivities(url, otpid):
+def getLocalApprovalActivities(url, policy_relationship_map):
-    pass
 
-    """
+    localapprovals = getLocalApprovals(url)
-def monitorOTP(url, pups):
     
-    getLocalApproval(url)
+    newly_added, same_policy, moved_to_audit, moved_to_enforcement, unusual_move = monitorAuditStatus(url, policy_relationship_map)
+
+    register_function("add_hash_and_return_enforcement", returnFromLocalApproval)
+
     
 
-    old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
-    new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
 
-    if os.path.exists(old_otp_path):
-        old_active_OTP = pd.read_parquet(old_otp_path)
-    else:
-        old_active_OTP = pd.DataFrame(columns=['otpid'])  # Ensure expected column exists
 
-    current_active_OTP = pd.read_parquet(new_otp_path)
+ 
-
+  
-    
+     
-    if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
+       #run_once_job(f" ", "addhash", time.time() + early, [url, clientid, pid, pups], None)
-    if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
-
-    newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
-    still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
-    no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
-    
-
-    register_function("addhash", addOTPHashes)
-
-    for _, row in newly_added.iterrows():
-        clientid = row['clientid']
-        duration = (int(row['duration']) * 60)
-        hostname = row['hostname']
-        purpose = row ['purpose']
-        pid = row['otpid']
-        early = math.floor(duration * .95)
-        #If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
-        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
-        print(f"Processing: {pid} with other data: {row}")
         
+def returnFromLocalApproval(url, device_df, policy_relationship_map, bad_publisher_list, pups, threat_tolerance_constant):
+    # Get unique policy names from device list
+    policies_in_devicelist = sorted(device_df['policy_name'].unique().tolist())
 
-    for _, row in still_in_OTP.iterrows():
+    # Create inverse map to go from Audit to Enforcement
-        pid = row['otpid']
+    inverse_map = {v: k for k, v in policy_relationship_map.items()}
-        #While still in OTP, continue to update activities list
-        getOTPActivities(url,pid)
 
-    for _, row in no_longer_OTP.iterrows():
+    # Fetch all policies 
-        clientid = row['clientid']
+    all_policies = policyf.getPolicyDataframe(url)
-        hostname = row['hostname']
-        purpose = row ['purpose']
-        pid = row['otpid']
-        allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
-        policy = clientf.getPolicyFromClientID(url,clientid)
-       
-        """
-        #Devices can come out of OTP either by timeout, or by early move out of OTP.  If they are manually moved out prior to the job to add hashes can run, we want to accelerate the job. 
-       # But first, we want to update the OTP activities one final time for the pid, then move up any jobs if they exist, then add the hashes to the local approval allowlist
-    """
-        
-        getOTPActivities(url,pid)
-        find_and_prioritize_jobs_by_pid(pid, 1)        
-        addOTPHashes(url, clientid,pid, pups)
-        
-        finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
-        finalhashesadded['policy'] = policy
-        finalhashesadded['allowlist'] = allowlist
-        finalhashesadded['added_at'] = time.localtime()
-        
-        if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"): 
-            df = pd.DataFrame()
-            df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
-        
-        history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
-        history = pd.concat([history, finalhashesadded], ignore_index=True)
-        ct.style_dataframe_dark(history, f"localapproval_history.html")
 
-        history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+    # Define policy types
-        
+    policy_types = [1, 2, 6, 7]
-        os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
-        del finalhashesadded
 
-def addOTPHashes(url, clientid, otpid, pups):
+    # Define output directories
-    path = f"OTP\\PARQ\\otp_activities_{otpid}.parquet"
+    parq_base_dir = "Local_Approval\\parquet\\"
-    activities = pd.read_parquet(path)
+    needappr_base_dir = "Local_Approval\\needs_approved"
 
-    pattern = pathf.regulator(pups)
+    # Ensure output directories exist
-    allowlist = clientf.getDestAllowlistFromClientID(url, clientid)
+    os.makedirs(parq_base_dir, exist_ok=True)
+    os.makedirs(needappr_base_dir, exist_ok=True)
 
-    # Initialize or preserve 'hash_added' column
+    # Build execution history
-    if "hash_added" not in activities.columns: activities["hash_added"] = None
+    policyf.buildExecHistory(
-
+        url,
-    # Identify rows that should be added (not matching pattern and not already added)
+        policies_in_devicelist,
-    approve_by_hash = activities[
+        parq_base_dir,
-        ~activities["filename"].str.contains(pattern, na=False) & (activities["hash_added"] != "added")
+        needappr_base_dir,
-    ]
+        policy_types,
-
+        threat_tolerance_constant,
-    hashes_to_add = approve_by_hash["sha256"].tolist()
+        bad_publisher_list,
-
+        pups,
-    # Add hashes to policy
-    if hashes_to_add:
-        policyf.addHash(url, allowlist, hashes_to_add)
-
-    # Update 'hash_added' column
-    activities["hash_added"] = activities.apply(
-        lambda row: "do not add" if pd.notna(row["filename"]) and pattern in row["filename"]
-        else ("added" if row["sha256"] in hashes_to_add else row["hash_added"]),
-        axis=1
     )
 
-    # Save the updated DataFrame
+    # Read hash data
-    activities.to_parquet(path)
+    unknown_hashes = ct.tryToReadCSV(f"{needappr_base_dir}unknown_hashes.csv")
-    """
+    good_hashes = ct.tryToReadCSV(f"{needappr_base_dir}good_hashes.csv")
+    hashes = pd.concat([unknown_hashes, good_hashes], ignore_index=True)
 
-def generateLocalApprovals(url):
+    # Process each policy
+    for policy in policies_in_devicelist:
+        # Filter for matching policy
+        matching_rows = all_policies[all_policies['name'] == policy]
+        
+        if matching_rows.empty:
+            print(f"Warning: No group ID found for policy '{policy}'. Skipping.")
+            continue
+
+        # Extract group ID
+        policy_id = matching_rows['groupid'].values[0]
+
+        # Map to destination ID
+        destination_id = inverse_map.get(policy_id)
+        if destination_id is None:
+            print(f"Warning: No corresponding enforcement policy found for group ID '{policy_id}'. Skipping.")
+            continue
+
+        allowlist = policyf.getDestAllowlist(url, destination_id)
+        
+        policyf.addHash(url, allowlist, hashes[hashes['group'] == policy_id])
+        devices_to_move = device_df[device_df['groupid'] == policy_id]
+        devices = devices_to_move['agentid'].drop_duplicates().tolist()
+        for device in devices:
+            clientf.moveAgentToEnforcement(url, device, policy_relationship_map)
+        
+def moveToLocalApproval(url, policy_relationship_map):
 
     possible_durations = [15, 60, 360, 1440, 10080]
     duration_selected = None
@@ -183,10 +156,10 @@ def generateLocalApprovals(url):
             duration_selected = possible_durations[choice - 1]
             print(ct.colorText(f"You selected: {duration_selected}", "yellow"))
         else:
-            print(ct.colorText("Invalid choice.", "red"))
+            print(ct.colorText("❌ Invalid choice.", "red"))
             return
     except ValueError:
-        print(ct.colorText("Invalid input. Please enter a number.", "red"))
+        print(ct.colorText("❌ Invalid input. Please enter a number.", "red"))
         return
 
     devicelist = clientf.promptForDevices()
@@ -194,14 +167,19 @@ def generateLocalApprovals(url):
 
     batch = int(time.time())
 
+    if device_df is None or device_df.empty:
+        print(ct.colorText("❌ No agents found or error retrieving agents.", "red"))
+        return
+
     for row in device_df.itertuples(index=False):
         try:
-            localapproval(url, batch, duration_selected, row.agentid)
+            addLocalApproval(url, batch, duration_selected, row.agentid)
+            clientf.moveAgentToAudit(url, row.agentid, policy_relationship_map)
         except Exception as e:
             print(ct.colorText(f"❌ Error processing agent {row.agentid}: {e}", "red"))
 
-def localapproval(url, batchid, duration_selected, agentid):
+def addLocalApproval(url, batchid, duration_selected, agentid):
-    purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - starting at {batchid} - for {agentid}"
+    purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - batch:{batchid} Client:{agentid}"
     endpoint = url + '/v1/otp/retrieve'
     payload = {
         "duration": str(duration_selected),
@@ -219,78 +197,86 @@ def localapproval(url, batchid, duration_selected, agentid):
     except Exception as e: 
         print(ct.colorText(f"An unexpected error occurred for agent {agentid}: {str(e)}", "red"))
         
-"""
+def monitorAuditStatus(url: str, policy_relationship_map: dict):
-def monitorOTP(url, pups):
-    
-    getActiveOTP(url)
     
+    # Simulated current agent list
+    current_agent_list = clientf.findAllAgents(url)
 
-    old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
+    # Load old agent list
-    new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
+    old_agent_path = "Local_Approval\\PARQ\\last_agent_list.parquet"
-
+    if os.path.exists(old_agent_path):
-    if os.path.exists(old_otp_path):
+        old_agent_list = pd.read_parquet(old_agent_path)
-        old_active_OTP = pd.read_parquet(old_otp_path)
     else:
-        old_active_OTP = pd.DataFrame(columns=['otpid'])  # Ensure expected column exists
+        old_agent_list = pd.DataFrame(columns=current_agent_list.columns)
 
-    current_active_OTP = pd.read_parquet(new_otp_path)
+    # Merge on hostname
+    merged = pd.merge(
+        old_agent_list[['hostname', 'groupid']],
+        current_agent_list[['hostname', 'groupid']],
+        on='hostname',
+        how='outer',
+        suffixes=('_old', '_current'),
+        indicator=True
+    )
 
-    
+    # Reverse map for enforcement
-    if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
+    reverse_policy_map = {v: k for k, v in policy_relationship_map.items()}
-    if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
+    known_transitions = set(policy_relationship_map.items()) | set(reverse_policy_map.items())
 
-    newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
+    # 1. Newly added
-    still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
+    newly_added = merged[merged['_merge'] == 'right_only']
-    no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
-    
 
-    register_function("addhash", addOTPHashes)
+    # 2. Same policy
+    same_policy = merged[
+        (merged['_merge'] == 'both') &
+        (merged['groupid_old'] == merged['groupid_current'])
+    ]
 
-    for _, row in newly_added.iterrows():
+    # 3. Moved to audit
-        clientid = row['clientid']
+    moved_to_audit = merged[
-        duration = (int(row['duration']) * 60)
+        (merged['_merge'] == 'both') &
-        hostname = row['hostname']
+        (merged['groupid_old'].isin(policy_relationship_map)) &
-        purpose = row ['purpose']
+        (merged['groupid_current'] == merged['groupid_old'].map(policy_relationship_map))
-        pid = row['otpid']
+    ]
-        early = math.floor(duration * .95)
-        #If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
-        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
-        print(f"Processing: {pid} with other data: {row}")
-        
 
-    for _, row in still_in_OTP.iterrows():
+    # 4. Moved to enforcement
-        pid = row['otpid']
+    moved_to_enforcement = merged[
-        #While still in OTP, continue to update activities list
+        (merged['_merge'] == 'both') &
-        getOTPActivities(url,pid)
+        (merged['groupid_old'].isin(reverse_policy_map)) &
+        (merged['groupid_current'] == merged['groupid_old'].map(reverse_policy_map))
+    ]
 
-    for _, row in no_longer_OTP.iterrows():
+    # 5. Unusual moves
-        clientid = row['clientid']
+    unusual_move = merged[
-        hostname = row['hostname']
+        (merged['_merge'] == 'both') &
-        purpose = row ['purpose']
+        (merged['groupid_old'] != merged['groupid_current']) &
-        pid = row['otpid']
+        merged.apply(lambda row: (row['groupid_old'], row['groupid_current']) not in known_transitions, axis=1)
-        allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
+    ]
-        policy = clientf.getPolicyFromClientID(url,clientid)
-   
-        
-        getOTPActivities(url,pid)
-        find_and_prioritize_jobs_by_pid(pid, 1)        
-        addOTPHashes(url, clientid,pid, pups)
-        
-        finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
-        finalhashesadded['policy'] = policy
-        finalhashesadded['allowlist'] = allowlist
-        finalhashesadded['added_at'] = time.localtime()
-        
-        if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"): 
-            df = pd.DataFrame()
-            df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
-        
-        history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
-        history = pd.concat([history, finalhashesadded], ignore_index=True)
-        ct.style_dataframe_dark(history, f"localapproval_history.html")
 
-        history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+    current_agent_list.to_parquet("Local_Approval\\PARQ\\last_agent_list.parquet", index=False)
-        
+
-        os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
+    # Return all five DataFrames
-        del finalhashesadded
+    return newly_added, same_policy, moved_to_audit, moved_to_enforcement, unusual_move
-"""
+
+def getNewLocalApprovals(url):
+    current_la = getLocalApprovals(url)
+
+    # Load old approval list
+    old_la_path = "Local_Approval\\PARQ\\last_la.parquet"
+    if os.path.exists(old_la_path):
+        old_la = pd.read_parquet(old_la_path)
+    else:
+        old_la = pd.DataFrame(columns=current_la.columns)
+
+    # Create composite keys
+    current_la['key'] = current_la['clientid'].astype(str) + "_" + current_la['granted'].astype(str)
+    old_la['key'] = old_la['clientid'].astype(str) + "_" + old_la['granted'].astype(str)
+
+    # Find new entries
+    new_entries = current_la[~current_la['key'].isin(old_la['key'])]
+
+    # Save current approvals for next run
+    current_la.drop(columns=['key'], inplace=True)
+    current_la.to_parquet(old_la_path, index=False)
+
+    return new_entries 

utils/otpfunctions.py

@@ -128,7 +128,7 @@ def monitorOTP(url, pups):
         purpose = row ['purpose']
         pid = row['otpid']
         allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
-        policy = clientf.getPolicyFromClientID(url,clientid)
+        policy, policyid = clientf.getPolicyFromClientID(url,clientid)
        
         """
         Devices can come out of OTP either by timeout, or by early move out of OTP.  If they are manually moved out prior to the job to add hashes can run, we want to accelerate the job. 

utils/policyfunctions.py

@@ -235,7 +235,7 @@ def checkpoint_stomper(checkpoint, url, type, policy, headers):
     parse_text = json.loads(json.dumps(json_output))
     return parse_text
 
-def listPolicies(url): 
+def listPolicies(url):
     endpoint = url + '/v1/group'
     print(ct.colorText("[+] Grabbing All Policies", "cyan"))
     payload = {}
@@ -244,9 +244,13 @@ def listPolicies(url):
     }
     response = requests.request("POST", endpoint, headers=headers, data=payload, verify=False)
     parse_text = json.loads(response.text)
+    return parse_text
+
+def choosePolicies(url): 
+    allpolicies = listPolicies(url)
     policiesnames = []
     policyids = []
-    for index, list in enumerate(parse_text['response']['groups'], start=1):
+    for index, list in enumerate(allpolicies['response']['groups'], start=1):
         print(ct.colorText(f"{index}. {list['name']}", "yellow"))
         policiesnames.append(list['name'])
         policyids.append(list['groupid'])
@@ -254,6 +258,9 @@ def listPolicies(url):
     choice = int(choice) - 1
     return choice, policiesnames, policyids
 
+def getPolicyDataframe(url) -> pd.DataFrame:
+    return pd.DataFrame(listPolicies(url)['response']['groups'])
+
 def listATPolicies(url):
     endpoint = url + '/v1/group'
     print(ct.colorText("[+] Grabbing All Policies", "cyan"))
@@ -440,7 +447,7 @@ def prepare_to_enforce(url, bad_publisher_list, pups, badpathparts, threat_toler
         elif choice == "2":
     
             print(ct.colorText(f"Please choose destination_name Policy for Path Exclusions","white"))
-            choice, policynames, policyid = listPolicies(url)
+            choice, policynames, policyid = choosePolicies(url)
             #print(allowlist_parent_tuple)
             destination_name = policynames[choice]
             destination_id = policyid[choice]
@@ -590,6 +597,7 @@ def savePreFlights(appr_base_dir, parq_base_dir, pflight_base_dir):
                 ct.style_dataframe_dark(df, f"{pflight_base_dir}{name}.html")
                 
 def getMultiplePolicySelections(url):
+    
     policynameslist = []
     policyidlist = []
     while True:
@@ -617,4 +625,32 @@ def getMultiplePolicySelections(url):
             break
 
     
-    return policynameslist, policyidlist
+    return policynameslist, policyidlist
+
+def getDestAllowlist(url, groupid):
+    
+    allowlists = getPolicyAllowlists(url, groupid) 
+ 
+    matches = allowlists.loc[
+        allowlists['name'].str.contains('local', case=False, na=False) &
+        allowlists['name'].str.contains('approval', case=False, na=False),
+        'applicationid'
+    ].values
+    app_id = matches[0] if len(matches) > 0 else None
+    
+    return app_id
+
+def getPolicyAllowlists(url, groupid):
+
+    endpoint = url + '/v1/group/policies'
+    payload = {
+            "groupid" : f"{groupid}"
+    }
+
+    headers = {"X-APIKey": os.getenv('APIKEY')}
+    payload = json.dumps(payload)
+
+    response = requests.post(endpoint, headers=headers, data=payload, verify=False)
+    result = json.loads(response.text)
+    data = pd.DataFrame(result["response"]["applications"])
+    return data