=
133 lines
4.2 KiB
Python
133 lines
4.2 KiB
Python
import pandas as pd
|
|
import requests
|
|
import os
|
|
import json
|
|
|
|
|
|
def aggregateHashes(executions_json) -> pd.DataFrame:
|
|
"""
|
|
Takes the executions, aggregates all the data with sha256 as primary, then returns aggregated dataframe
|
|
"""
|
|
data = executions_json.json()
|
|
df = pd.DataFrame(data["response"]["exechistories"])
|
|
|
|
if df.empty:
|
|
return df
|
|
|
|
# Aggregate by sha256, deduplicate lists, and preserve order
|
|
agg_df = df.groupby("sha256").agg(lambda x: list(dict.fromkeys(x))).reset_index()
|
|
|
|
# Add a column for the number of unique hostnames
|
|
agg_df["num_devices"] = agg_df["hostname"].apply(len)
|
|
|
|
# Sort by num_devices in descending order
|
|
agg_df = agg_df.sort_values("num_devices", ascending=False)
|
|
|
|
return agg_df
|
|
|
|
def augmentAggregatedHashes(url, agg_df: pd.DataFrame) -> pd.DataFrame:
|
|
"""
|
|
Takes output of aggregatedHashes, queries API for those hashes, flattens response while keeping one row per hash,
|
|
aggregate applications and baselines into lists, then merges results back into agg_df to create a
|
|
"""
|
|
endpoint = url + '/v1/hash/query'
|
|
payload = {
|
|
"hashes": agg_df['sha256'].tolist()
|
|
}
|
|
|
|
headers = {"X-APIKey": os.getenv('APIKEY')}
|
|
payload = json.dumps(payload)
|
|
|
|
response = requests.post(endpoint, headers=headers, data=payload, verify=False)
|
|
data = response.json()
|
|
results = data.get("response", {}).get("results", [])
|
|
|
|
rows = []
|
|
for res in results:
|
|
row = {"sha256": res.get("sha256"), "result": res.get("result")}
|
|
|
|
if "data" in res:
|
|
d = res["data"]
|
|
for key in ["filename", "filepath", "description", "filesize", "md5",
|
|
"productname", "productversion", "publisher", "createtime", "modtime",
|
|
"sha128", "sha384", "sha512", "datetime"]:
|
|
row[key] = d.get(key)
|
|
|
|
row["applications"] = d.get("applications", [])
|
|
row["baselines"] = d.get("baselines", [])
|
|
|
|
reputation = d.get("reputation", {})
|
|
for k, v in reputation.items():
|
|
row[f"reputation_{k}"] = v
|
|
|
|
rows.append(row)
|
|
|
|
df_api = pd.DataFrame(rows)
|
|
|
|
aug_df = agg_df.merge(df_api, on="sha256", how="left")
|
|
|
|
return aug_df
|
|
|
|
def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
|
|
if untrusted_publishers is None:
|
|
untrusted_publishers = []
|
|
|
|
df = aug_df.copy()
|
|
def reputationtool(row, threat_tolerance):
|
|
if row["reputation_scannermatch"] == "N/A":
|
|
return True
|
|
try:
|
|
if int(row["reputation_scannermatch"]) > threat_tolerance:
|
|
return True
|
|
except (ValueError, TypeError):
|
|
pass
|
|
return False
|
|
|
|
mask_needsreview = (df["publisher_y"] == "Not Signed") & df.apply(lambda row: reputationtool(row, threat_tolerance), axis=1)
|
|
mask_approved = (df["publisher_y"] != "Not Signed") & (~df["publisher_y"].isin(untrusted_publishers))
|
|
|
|
needsreview_df = df[mask_needsreview]
|
|
approved_df = df[mask_approved]
|
|
remaining_df = df[~(mask_needsreview | mask_approved)]
|
|
|
|
return needsreview_df, approved_df, remaining_df
|
|
|
|
|
|
"""
|
|
def categorizeHashes(aug_df: pd.DataFrame, threat_tolerance: int, untrusted_publishers: list):
|
|
|
|
Categorize hashes into needsreview, approved, and remaining based on publisher and threat level.
|
|
|
|
|
|
if untrusted_publishers is None:
|
|
untrusted_publishers = []
|
|
|
|
# Flatten threatlevel from nested reputation dict
|
|
df = aug_df.copy()
|
|
|
|
# Masks for each category
|
|
mask_needsreview = ((df["publisher_y"] == "Not Signed") & reputationtool(df))
|
|
|
|
print(mask_needsreview)
|
|
|
|
mask_approved = (df["publisher_y"] != "Not Signed") & (~df["publisher_y"].isin(untrusted_publishers))
|
|
|
|
print(mask_approved)
|
|
|
|
# Create DataFrames for each category
|
|
needsreview_df = df[mask_needsreview]
|
|
approved_df = df[mask_approved]
|
|
remaining_df = df[~(mask_needsreview | mask_approved)]
|
|
|
|
return needsreview_df, approved_df, remaining_df
|
|
|
|
def approvehashes(approved_df: pd.DataFrame):
|
|
pass
|
|
|
|
def reputationtool(df):
|
|
if df["reputation_scannermatch"] == "N/A":
|
|
return True
|
|
if df["reputation_scannermatch"].astype(int) > 3:
|
|
return True
|
|
return False
|
|
""" |