Scheduler added

utils/otpfunctions.py

@@ -4,6 +4,15 @@ import os
 import pandas as pd
 import utils.pretty as ct
 import shutil
+import math
+import time
+import utils.clientfunctions as clientf
+import utils.pathfunctions as pathf
+import utils.policyfunctions as policyf
+from utils.perstscheduler import register_function, run_once_job, recurring_job, reload_jobs, start_scheduler
+
+
+
 
 
 def getActiveOTP(url):
@@ -19,45 +28,105 @@ def getActiveOTP(url):
     response = requests.post(endpoint, headers=headers, data=payload, verify=False)
     result = json.loads(response.text)
     otp = pd.DataFrame(result["response"]["otpusage"])
-    otp.to_parquet("parquet\\newest_active_OTP.parquet", index=False)
+    otp.to_parquet("OTP\\PARQ\\newest_active_OTP.parquet", index=False)
     if not otp.empty:
         ct.style_dataframe_dark(otp, f"newest_active_OTP.html")
    
 
-
 def getOTPActivities(url, otpid):
-
     endpoint = url + f'/v1/otp/activities'
-    payload = {
-            "otpid" : f"{otpid}"
-    }
-
+    payload = {"otpid": f"{otpid}"}
     headers = {"X-APIKey": os.getenv('APIKEY')}
     payload = json.dumps(payload)
 
     response = requests.post(endpoint, headers=headers, data=payload, verify=False)
     result = json.loads(response.text)
-    otp = pd.DataFrame(result["response"]["otpactivities"])
-    otp.to_parquet("parquet/otp_activities.parquet", index=False)
-    if not otp.empty:
-        ct.style_dataframe_dark(otp, f"OTP_activities_{otpid}.html")
+    new_data = pd.DataFrame(result["response"]["otpactivities"])
 
+    # Define file path
+    parquet_path = f"OTP\\PARQ\\otp_activities_{otpid}.parquet"
 
-def compareOTPHistory():
+    # Check if file exists and read it
+    if os.path.exists(parquet_path):
+        existing_data = pd.read_parquet(parquet_path)
+        combined_data = pd.concat([existing_data, new_data], ignore_index=True)
+        combined_data.drop_duplicates(inplace=True)
+    else:
+        combined_data = new_data
 
-    if not os.path.exists("parquet\\old_active_OTP"): 
-        shutil.copy2("parquet\\newest_active_OTP.parquet", "parquet\\old_active_OTP.parquet")
+    # Save combined data
+    combined_data.to_parquet(parquet_path, index=False)
 
-    old_active_OTP = pd.read_parquet("parquet\\old_active_OTP.parquet")
-    current_active_OTP = pd.read_parquet("parquet\\newest_active_OTP.parquet")
+    # Optional: generate styled HTML if there's data
+    if not combined_data.empty:
+        ct.style_dataframe_dark(combined_data, f"OTP/HTML/OTP_activities_{otpid}.html")
+
+def monitorOTP(url, pups):
+    
+    getActiveOTP(url)
+    
+    if not os.path.exists("OTP\\old_active_OTP"): 
+        shutil.copy2("OTP\\PARQ\\newest_active_OTP.parquet", "OTP\\PARQ\\old_active_OTP.parquet")
+
+    old_active_OTP = pd.read_parquet("OTP\\PARQ\\old_active_OTP.parquet")
+    current_active_OTP = pd.read_parquet("OTP\\PARQ\\newest_active_OTP.parquet")
 
     newly_added = current_active_OTP[~current_active_OTP['otpid'].isin(old_active_OTP['otpid'])]
     still_in_OTP = old_active_OTP[old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
     no_longer_OTP = old_active_OTP[~old_active_OTP['otpid'].isin(current_active_OTP['otpid'])]
 
-    
-  
+    register_function("addhash", addOTPHashes)
+
     for _, row in newly_added.iterrows():
+        clientid = row['clientid']
+        duration = (row['duration'] * 60)
+        hostname = row['hostname']
+        purpose = row ['purpose']
+        pid = row['otpid']
+        early = math.floor(duration * .9)
+
+        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + early, args=[url, clientid, pid, pups])
+        run_once_job(f"Add activity hashes for {pid}, for {hostname} for the purpose: {purpose}", "addhash", time.time() + duration, args=[url, clientid, pid, pups])
+        print(f"Processing: {pid} with other data: {row}")
+        
+
+    for _, row in still_in_OTP.iterrows():
+        pid = row['otpid']
+        getOTPActivities(url,pid)
+
+    for _, row in no_longer_OTP.iterrows():
         pid = row['otpid']
         # Access other columns via row['column_name']
         print(f"Processing PID: {pid} with other data: {row}")
+
+
+def addOTPHashes(url, clientid, otpid, pups):
+    path = f"OTP\\PARQ\\otp_activities_{otpid}.parquet"
+    activities = pd.read_parquet(path)
+
+    pattern = pathf.regulator(pups)
+    policy = clientf.getDestAllowlistFromClientID(clientid)
+
+    # Initialize or preserve 'hash_added' column
+    if "hash_added" not in activities.columns: activities["hash_added"] = None
+
+    # Identify rows that should be added (not matching pattern and not already added)
+    approve_by_hash = activities[
+        ~activities["filename"].str.contains(pattern, na=False) & (activities["hash_added"] != "added")
+    ]
+
+    hashes_to_add = approve_by_hash["sha256"].tolist()
+
+    # Add hashes to policy
+    if hashes_to_add:
+        policyf.addHash(url, policy, hashes_to_add)
+
+    # Update 'hash_added' column
+    activities["hash_added"] = activities.apply(
+        lambda row: "do not add" if pd.notna(row["filename"]) and pattern in row["filename"]
+        else ("added" if row["sha256"] in hashes_to_add else row["hash_added"]),
+        axis=1
+    )
+
+    # Save the updated DataFrame
+    activities.to_parquet(path)