AirlockTools/utils/pathfunctions.py

import pandas as pd
import os
from itertools import chain

def filepathInitialGroup(df: pd.DataFrame):
    original_columns = df.columns.tolist()

    # Step 1: Split comma-separated filepaths into lists
    df["filename_x"] = df["filename_x"].str.split(",")

    # Step 2: Explode the list so each filepath becomes its own row
    df = df.explode("filename_x", ignore_index=True)

    # Step 3: Clean up whitespace
    df["filename_x"] = df["filename_x"].str.strip()

    # Step 4: Extract directory and filename from each filepath
    df["directory"] = df["filename_x"].apply(lambda x: os.path.dirname(x) if pd.notna(x) else "")
    df["filename"] = df["filename_x"].apply(lambda x: os.path.basename(x) if pd.notna(x) else "")

    # Step 5: Drop the original raw filepath column
    df = df.drop(columns=["filename_x"])

    # Helper functions for path manipulation
    def get_parts(path):
        return path.strip("\\").split("\\")

    def join_parts(parts):
        return "\\".join(parts)

    def longest_common_prefix(paths):
        split_paths = [get_parts(p) for p in paths]
        min_len = min(len(p) for p in split_paths)
        prefix = []
        for i in range(min_len):
            current = split_paths[0][i]
            if all(p[i] == current for p in split_paths):
                prefix.append(current)
            else:
                break
        return join_parts(prefix)

    # Step 6: Group directories by shared prefix using custom logic
    """
    Loop through each directory path
    directories: list of all directory paths.
    groups: will hold lists of grouped directories.
    used: tracks which directories have already been grouped.
    """
    directories = df["directory"].tolist()
    groups = []
    used = set()

    #For Each directory, compare it with others
    """
    Skip if already grouped.
    Start a new group with the current path.
    parts_i is the list of folder names in the path (e.g., ["C:", "Users", "John", "Documents"]).
    """
    
    for i, path in enumerate(directories):
        if path in used:
            continue
        group = [path]
        parts_i = get_parts(path)

    #Compare with all other directories: For each other directory, split it into parts and find the common prefix (shared folder structure).
        """
        Logic:
        If the directory is deep (>3 parts) and shares at least 3 parts → group it.
        If it's exactly 3 parts long and shares at least 2 → group it.
        Or, if it shares all but one part and is deep → group it.
        These rules are designed to:
        Group directories that are closely related in structure.
        Avoid grouping unrelated paths that just happen to start similarly.
        """  

        for j in range(i + 1, len(directories)):
            parts_j = get_parts(directories[j])
            common = os.path.commonprefix([parts_i, parts_j])
        #Apply grouping rules
            if (len(parts_i) > 3 and len(common) >= 3) or (len(parts_i) == 3 and len(common) >= 2):
                group.append(directories[j])
                used.add(directories[j])
            elif len(common) == len(parts_i) - 1 and len(parts_i) > 3:
                group.append(directories[j])
                used.add(directories[j])
        used.add(path)
        groups.append(group)

    # Step 7: Map each original directory to its grouped prefix
    prefix_map = {dir: longest_common_prefix(group) for group in groups for dir in group}
    df["grouped_directory"] = df["directory"].map(prefix_map)

    # Step 8: Group the DataFrame by grouped_directory
    aggregation = {col: (lambda x: list(x)) for col in original_columns if col not in ["filename_x"]}
    aggregation.update({
        "directory": lambda x: list(x),
        "filename": lambda x: list(x)
    })

    grouped_df = df.groupby("grouped_directory", as_index=False).agg(aggregation)

    # Step 9: Split into eligible and ineligible paths based on depth
    grouped_df["depth"] = grouped_df["grouped_directory"].apply(lambda x: len(get_parts(x)))
    path_eligible = grouped_df[grouped_df["depth"] > 2].drop(columns=["depth"])
    path_ineligible = grouped_df[grouped_df["depth"] <= 2].drop(columns=["depth"])

    # Step 10: Move entries from eligible to ineligible if grouped_directory contains 'C:\Users' or 'c$\Users'
    mask = path_eligible["grouped_directory"].str.contains(r"(?i)(?:\\Users|\\c\$\\Users|inetpub\\wwwroot|windows\\temp)", na=False)
    move_to_ineligible = path_eligible[mask]
    path_eligible = path_eligible[~mask]
    path_ineligible = pd.concat([path_ineligible, move_to_ineligible], ignore_index=True)

    # Step 11: Deduplicate list elements in all columns
    def deduplicate_lists(df):
        for col in df.columns:
            if df[col].apply(lambda x: isinstance(x, list)).all():
                df[col] = df[col].apply(lambda x: list({str(item): item for item in chain.from_iterable(x if isinstance(x[0], list) else [x])}.values()))
        return df


    path_eligible = deduplicate_lists(path_eligible)
    path_ineligible = deduplicate_lists(path_ineligible)

    return path_eligible, path_ineligible