Enhanced Quiet Finder

AirlockTools.py

@@ -17,6 +17,7 @@ import dotenv
 import os
 import urllib3
 import utils.clientfunctions
+import utils.localapproval as la
 import utils.otpfunctions
 import utils.policyfunctions
 import utils.utils as ct
@@ -39,8 +40,8 @@ threat_tolerance_constant = 4
 policy_relationship_map ={ #Enforcement : Audit
 "bf0b1f9b-bfea-4f44-97c0-80e27ff61712" : "538d3218-92f4-4943-a6ee-db9267ab62d8", #AT Servers General, AT Servers General Audit
 "31ababac-65de-4c6a-86dd-6691d7e3ee3b" : "fc05b42a-b846-4e72-88ca-c35d416e699f", #AT Epic, #AT Epic Audit
-"d1f58960-f866-49e0-848a-a5b09fffd4cd" : "d55c03a6-c376-4391-8626-4f843b882a7c" #AT DMZ Enforced, #AT DMZ Audit
- 
+"d1f58960-f866-49e0-848a-a5b09fffd4cd" : "d55c03a6-c376-4391-8626-4f843b882a7c", #AT DMZ Enforced, #AT DMZ Audit
+ "504dd011-86b6-489a-b78f-eff589cef8aa" : "88a1cfdc-3b30-448b-b309-be16fe437ca3" #AT Workstations BCA, #AT Workstations BCA Audit3
 }
  
 def main():
@@ -57,14 +58,17 @@ def main():
             os.makedirs("scheduling", exist_ok=True)
             os.makedirs("OTP/HTML", exist_ok=True)
             os.makedirs("OTP/PARQ", exist_ok=True)
-
+            os.makedirs("Local_Approval/HTML", exist_ok=True)
+            os.makedirs("Local_Approval/PARQ", exist_ok=True)
             ct.apivalidation()
                     
             register_function("monitorOTP", utils.otpfunctions.monitorOTP)
             register_function("updateAudit", utils.policyfunctions.updateAuditPoliciesFromEnforcementPolices)
+            register_function("monitorLA", la.monitorLA)
             
             if not os.path.exists("scheduling\\jobs.json"): 
                 recurring_job("monitorOTP", "monitorOTP", interval=60, unit="seconds", args=[url, pups])
+                recurring_job("monitorLA", "monitorLA", interval=45, unit="seconds", args=[url, pups])
                 recurring_job("updateAudit", "updateAudit", interval=10, unit="minutes", args=[url, policy_relationship_map])
             else:
                 reload_jobs()
@@ -104,9 +108,10 @@ def menu_main():
             utils.clientfunctions.findAgents(url, devicelist, False)
 
         elif choice == "7":
-            devicelist = utils.clientfunctions.promptForDevices()
-            device_df = utils.clientfunctions.findAgents(url, devicelist, True)
-            utils.clientfunctions.returnToEnforcement(url, device_df, policy_relationship_map, bad_publisher_list, pups, badpathparts, threat_tolerance_constant, path_exclusion_constant, min_files_for_path)
+            la.generateLocalApprovals(url)
+
+        elif choice == "8":
+            p
         elif choice == "Q":
             break
         else:

utils/clientfunctions.py

@@ -238,13 +238,14 @@ def findAgents(url, device_input_str, return_dataframe):
     #Filter the DataFrame using regex
     matched_df = df[df['hostname'].apply(lambda x: bool(regex.search(str(x))))]
     
+    if return_dataframe : return matched_df
+    else:
         #Export to CSV with timestamp
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
         filename = f"agentsearch_{timestamp}.csv"
         matched_df.to_csv(f"device_search\\{filename}", index=False)
 
         print(ct.colorText(f"\n✅ Matched devices exported to: device_search\\{filename}","green"))
-    if return_dataframe : return matched_df
     
 def promptForDevices():
     
@@ -351,16 +352,18 @@ def returnToEnforcement(url, device_df, policy_relationship_map, bad_publisher_l
             print(ct.colorText("Invalid choice. Please try again.", "red"))    
     """
         
-def findQuietAgents(url):
 
+def findQuietAgents(url):
+    # Get policy selection and agent list
     choice, policynames, policyid = policyf.listPolicies(url)
     policy = policynames[choice]
     groupid = policyid[choice]
     agents = findGroupAgents(url, groupid)
 
+    # Prompt user for history range
     while True:
         try:
-            history_days = int(input("Enter the how many days in of history do you want to pull - select a number between 1 and 150: "))
+            history_days = int(input("Enter how many days of history to pull (1–150): "))
             if 1 <= history_days <= 150:
                 break
             else:
@@ -368,36 +371,63 @@ def findQuietAgents(url):
         except ValueError:
             print("Invalid input. Please enter a valid integer.")
 
-    policy_exec_history = policyf.getPolicyInfo(url, policy, [1, 2, 6, 7], history_days)
+    while True:
+        try:
+            required_quiet = int(input("Enter how many days without an untrusted execution before these are considered ready for enforcement? (1–365): "))
+            if 1 <= required_quiet <= 365:
+                break
+            else:
+                print("Invalid input. Please enter a number between 1 and 365.")
+        except ValueError:
+            print("Invalid input. Please enter a valid integer.")
 
-    # Count occurrences of each hostname in the executions dataframe
+    # Get policy execution history
+    policy_exec_history = policyf.getPolicyInfo(url, policy, [1, 2, 6, 7], history_days, False)
+
+    # Convert 'datetime' column to timezone-aware datetime objects
+    policy_exec_history['datetime'] = pd.to_datetime(policy_exec_history['datetime'], format='%Y-%m-%dT%H:%M:%SZ', utc=True)
+
+    # Get current UTC time
+    now = datetime.datetime.now(datetime.timezone.utc)
+
+    # Calculate days ago
+    policy_exec_history['days_ago'] = policy_exec_history['datetime'].apply(lambda dt: (now - dt).days)
+
+    # Count total executions per hostname
     hostname_counts = policy_exec_history['hostname'].value_counts()
 
-    # Map those counts to the hostnames in the first dataframe
+    # Map execution counts to agents
     agents['execution_count'] = agents['hostname'].map(hostname_counts).fillna(0).astype(int)
 
+    # Find most recent execution per hostname
+    most_recent_exec = policy_exec_history.sort_values(by='days_ago').drop_duplicates(subset='hostname', keep='first')
+
+    # Map most recent execution age to agents
+    agents['days_since'] = agents['hostname'].map(most_recent_exec.set_index('hostname')['days_ago'])
+
+    #Check for enforcement readyness
+    agents['required_quiet'] = required_quiet
+    agents['enforce_ready'] = agents['days_since'].apply(
+        lambda x: True if pd.isna(x) or x > required_quiet else False
+    )
+
+    # Sort agents by execution count and hostname
     agents = agents.sort_values(by=['execution_count', 'hostname'], ascending=[True, True])
 
+    # Save to CSV
     print(ct.colorText(f"Saving CSV to {policy}_agents_last_{history_days}_days.csv", "green"))
     agents.to_csv(f"{policy}_agents_last_{history_days}_days.csv", index=False)
 
-    
-    # Count hosts with execution_count == 0
+    # Summary stats
     zero_count = (agents['execution_count'] == 0).sum()
-
-    # Total number of hosts
     total_hosts = len(agents)
-
-    # Calculate percentage
     zero_percentage = (zero_count / total_hosts) * 100
 
-    # Print results
     print(f"Number of hosts in policy: {total_hosts}")
     print(f"Number of hosts with execution_count = 0: {zero_count}")
     print(f"Percentage of hosts with execution_count = 0: {zero_percentage:.2f}%")
 
 
-
 def findGroupAgents(url, groupid):        
     endpoint = url + '/v1/agent/find'
     payload = {

utils/localapproval.py

@@ -0,0 +1,296 @@
+#    Copyright (C) 2025 James Brotosky, Brandon Wickline
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+#Local Imports
+import utils.clientfunctions as clientf
+import utils.pathfunctions as pathf
+import utils.policyfunctions as policyf
+import utils.utils as ct
+from utils.perstscheduler import register_function, run_once_job, recurring_job, reload_jobs, start_scheduler, find_and_prioritize_jobs_by_pid
+
+#Standard Libary Imports:
+import json
+import math
+import os
+import time
+
+#3rd Party Imports:
+import pandas as pd
+import requests
+
+def getLocalApprovals(url):
+
+    endpoint = url + f'/v1/otp/usage'
+    payload = {
+            "status" : "0"
+    }
+
+    headers = {"X-APIKey": os.getenv('APIKEY')}
+    payload = json.dumps(payload)
+
+    response = requests.post(endpoint, headers=headers, data=payload, verify=False)
+    result = json.loads(response.text)
+    local_approval = pd.DataFrame(result["response"]["otpusage"])
+    if os.path.exists("Local_Approval\\PARQ\\newest_local_approval.parquet"):
+        previous_run = pd.read_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet")
+        previous_run.to_parquet("Local_Approval\\PARQ\\last_local_approval.parquet", index=False)
+        os.remove("Local_Approval\\PARQ\\newest_local_approval.parquet")
+
+    #Only keep rows presumably created by the generate local approval function
+    local_approval = local_approval[local_approval['column_name'].str.startswith('🎫 Local Approval 🎫')]
+    local_approval.to_parquet("Local_Approval\\PARQ\\newest_local_approval.parquet", index=False)
+    if not local_approval.empty:
+        ct.style_dataframe_dark(local_approval, f"Local_Approval\\HTML\\newest_local_approval.html")
+   
+def getLocalApprovalActivities(url, otpid):
+    pass
+
+    """
+def monitorOTP(url, pups):
+    
+    getLocalApproval(url)
+    
+
+    old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
+    new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
+
+    if os.path.exists(old_otp_path):
+        old_active_OTP = pd.read_parquet(old_otp_path)
+    else:
+        old_active_OTP = pd.DataFrame(columns=['otpid'])  # Ensure expected column exists
+
+    current_active_OTP = pd.read_parquet(new_otp_path)
+
+    
+    if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
+    if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
+
+    newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
+    still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
+    no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
+    
+
+    register_function("addhash", addOTPHashes)
+
+    for _, row in newly_added.iterrows():
+        clientid = row['clientid']
+        duration = (int(row['duration']) * 60)
+        hostname = row['hostname']
+        purpose = row ['purpose']
+        pid = row['otpid']
+        early = math.floor(duration * .95)
+        #If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
+        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
+        print(f"Processing: {pid} with other data: {row}")
+        
+
+    for _, row in still_in_OTP.iterrows():
+        pid = row['otpid']
+        #While still in OTP, continue to update activities list
+        getOTPActivities(url,pid)
+
+    for _, row in no_longer_OTP.iterrows():
+        clientid = row['clientid']
+        hostname = row['hostname']
+        purpose = row ['purpose']
+        pid = row['otpid']
+        allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
+        policy = clientf.getPolicyFromClientID(url,clientid)
+       
+        """
+        #Devices can come out of OTP either by timeout, or by early move out of OTP.  If they are manually moved out prior to the job to add hashes can run, we want to accelerate the job. 
+       # But first, we want to update the OTP activities one final time for the pid, then move up any jobs if they exist, then add the hashes to the local approval allowlist
+    """
+        
+        getOTPActivities(url,pid)
+        find_and_prioritize_jobs_by_pid(pid, 1)        
+        addOTPHashes(url, clientid,pid, pups)
+        
+        finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
+        finalhashesadded['policy'] = policy
+        finalhashesadded['allowlist'] = allowlist
+        finalhashesadded['added_at'] = time.localtime()
+        
+        if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"): 
+            df = pd.DataFrame()
+            df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        
+        history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        history = pd.concat([history, finalhashesadded], ignore_index=True)
+        ct.style_dataframe_dark(history, f"localapproval_history.html")
+
+        history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        
+        os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
+        del finalhashesadded
+
+def addOTPHashes(url, clientid, otpid, pups):
+    path = f"OTP\\PARQ\\otp_activities_{otpid}.parquet"
+    activities = pd.read_parquet(path)
+
+    pattern = pathf.regulator(pups)
+    allowlist = clientf.getDestAllowlistFromClientID(url, clientid)
+
+    # Initialize or preserve 'hash_added' column
+    if "hash_added" not in activities.columns: activities["hash_added"] = None
+
+    # Identify rows that should be added (not matching pattern and not already added)
+    approve_by_hash = activities[
+        ~activities["filename"].str.contains(pattern, na=False) & (activities["hash_added"] != "added")
+    ]
+
+    hashes_to_add = approve_by_hash["sha256"].tolist()
+
+    # Add hashes to policy
+    if hashes_to_add:
+        policyf.addHash(url, allowlist, hashes_to_add)
+
+    # Update 'hash_added' column
+    activities["hash_added"] = activities.apply(
+        lambda row: "do not add" if pd.notna(row["filename"]) and pattern in row["filename"]
+        else ("added" if row["sha256"] in hashes_to_add else row["hash_added"]),
+        axis=1
+    )
+
+    # Save the updated DataFrame
+    activities.to_parquet(path)
+    """
+
+def generateLocalApprovals(url):
+
+    possible_durations = [15, 60, 360, 1440, 10080]
+    duration_selected = None
+
+    print(ct.colorText("Please select a duration:", "white"))
+    for i, option in enumerate(possible_durations, start=1):
+        print(f"{i}. {option}")
+
+    try:
+        choice = int(input("Enter the number of your choice: "))
+        if 1 <= choice <= len(possible_durations):
+            duration_selected = possible_durations[choice - 1]
+            print(ct.colorText(f"You selected: {duration_selected}", "yellow"))
+        else:
+            print(ct.colorText("Invalid choice.", "red"))
+            return
+    except ValueError:
+        print(ct.colorText("Invalid input. Please enter a number.", "red"))
+        return
+
+    devicelist = clientf.promptForDevices()
+    device_df = clientf.findAgents(url, devicelist, True)
+
+    batch = int(time.time())
+
+    for row in device_df.itertuples(index=False):
+        try:
+            localapproval(url, batch, duration_selected, row.agentid)
+        except Exception as e:
+            print(ct.colorText(f"❌ Error processing agent {row.agentid}: {e}", "red"))
+
+def localapproval(url, batchid, duration_selected, agentid):
+    purpose = f"🎫 Local Approval 🎫 - {duration_selected} mins - starting at {batchid} - for {agentid}"
+    endpoint = url + '/v1/otp/retrieve'
+    payload = {
+        "duration": str(duration_selected),
+        "agentid": str(agentid),
+        "purpose": purpose
+    }
+
+    headers = {"X-APIKey": os.getenv('APIKEY')}
+    response = requests.post(endpoint, headers=headers, data=json.dumps(payload), verify=False)
+
+    try:
+        result = response.json()
+        otpcode = result["response"]["otpcode"]
+        print(ct.colorText(f"The OTP code is: {otpcode}", "yellow"))
+    except Exception as e: 
+        print(ct.colorText(f"An unexpected error occurred for agent {agentid}: {str(e)}", "red"))
+        
+"""
+def monitorOTP(url, pups):
+    
+    getActiveOTP(url)
+    
+
+    old_otp_path = "OTP\\PARQ\\old_active_OTP.parquet"
+    new_otp_path = "OTP\\PARQ\\newest_active_OTP.parquet"
+
+    if os.path.exists(old_otp_path):
+        old_active_OTP = pd.read_parquet(old_otp_path)
+    else:
+        old_active_OTP = pd.DataFrame(columns=['otpid'])  # Ensure expected column exists
+
+    current_active_OTP = pd.read_parquet(new_otp_path)
+
+    
+    if 'otpid' not in current_active_OTP.columns: current_active_OTP = pd.DataFrame(columns=['otpid'])
+    if 'otpid' not in old_active_OTP.columns: old_active_OTP = pd.DataFrame(columns=['otpid'])
+
+    newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
+    still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
+    no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
+    
+
+    register_function("addhash", addOTPHashes)
+
+    for _, row in newly_added.iterrows():
+        clientid = row['clientid']
+        duration = (int(row['duration']) * 60)
+        hostname = row['hostname']
+        purpose = row ['purpose']
+        pid = row['otpid']
+        early = math.floor(duration * .95)
+        #If newly added to the list - schedule adding the majority of the executions prior to the expiration of OTP period.
+        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, [url, clientid, pid, pups], None)
+        print(f"Processing: {pid} with other data: {row}")
+        
+
+    for _, row in still_in_OTP.iterrows():
+        pid = row['otpid']
+        #While still in OTP, continue to update activities list
+        getOTPActivities(url,pid)
+
+    for _, row in no_longer_OTP.iterrows():
+        clientid = row['clientid']
+        hostname = row['hostname']
+        purpose = row ['purpose']
+        pid = row['otpid']
+        allowlist = clientf.getDestAllowlistFromClientID(url,clientid)
+        policy = clientf.getPolicyFromClientID(url,clientid)
+   
+        
+        getOTPActivities(url,pid)
+        find_and_prioritize_jobs_by_pid(pid, 1)        
+        addOTPHashes(url, clientid,pid, pups)
+        
+        finalhashesadded = pd.read_parquet(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
+        finalhashesadded['policy'] = policy
+        finalhashesadded['allowlist'] = allowlist
+        finalhashesadded['added_at'] = time.localtime()
+        
+        if not os.path.exists(f"OTP\\PARQ\\localapprovalhistory.parquet"): 
+            df = pd.DataFrame()
+            df.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        
+        history = pd.read_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        history = pd.concat([history, finalhashesadded], ignore_index=True)
+        ct.style_dataframe_dark(history, f"localapproval_history.html")
+
+        history.to_parquet(f"OTP\\PARQ\\localapprovalhistory.parquet")
+        
+        os.remove(f"OTP\\PARQ\\otp_activities_{pid}.parquet")
+        del finalhashesadded
+"""

utils/otpfunctions.py

@@ -223,3 +223,4 @@ def generateOTP(url, agentid):
     result = json.loads(response.text)
     otpcode = result["response"]["otpcode"]
     print(ct.colorText(f"The OPT code is: {otpcode}", "yellow"))
+

utils/policyfunctions.py

@@ -100,7 +100,7 @@ def getPolicyInfo(url, policy, type, days, parquet=True):
         data = json.loads(exehist)
         executionhist_policy = pd.DataFrame(data["response"]["exechistories"])
         if not executionhist_policy.empty:
-            executionhist_policy = executionhist_policy[['sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
+            executionhist_policy = executionhist_policy[['datetime','sha256', 'publisher', 'filename', 'hostname', 'username', 'pprocess', 'gprocess', 'commandline']]
             executionhist_policy['policy'] = policy  # Add policy column here
             executionhist_policy = executionhist_policy.drop_duplicates(subset=['sha256', 'filename', 'hostname'])
             executionhist_policy = executionhist_policy.sort_values(by=['sha256', 'filename'])
@@ -316,7 +316,6 @@ def listAllowlists(url: str) -> tuple[int, list, list]:
 def skipback(days):
     """
     Generate a MongoDB ObjectId for a given number of days ago from today.
-    Adds 1 extra day to the input to look further back.
     """
     adjusted_days = days
     date_days_ago = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=adjusted_days)