From a7b659c9515ccf4514008448a83471115da21495 Mon Sep 17 00:00:00 2001 From: Zarithas Date: Wed, 17 Dec 2025 12:38:31 -0500 Subject: [PATCH] policyprepworkflow: enhancements and fixes - Added row copy functionality (Ctrl+C) - Improved row selection visual contrast - Fixed data state issues when navigating between stages Server Log tab - Added new Server Log tab to the main application - Implemented live filtering with wildcard support - Enabled auto-refresh capability Multiagent selector - Added file loading support for device lists --- TUI/Screens/policyprepworkflowscreen.py | 751 +++++++++++++++++++++--- TUI/Widgets/multiagentselector.py | 132 ++++- TUI/Widgets/serverlogwidget.py | 110 +++- 3 files changed, 889 insertions(+), 104 deletions(-) diff --git a/TUI/Screens/policyprepworkflowscreen.py b/TUI/Screens/policyprepworkflowscreen.py index ec9b03a..3b068b8 100644 --- a/TUI/Screens/policyprepworkflowscreen.py +++ b/TUI/Screens/policyprepworkflowscreen.py @@ -33,7 +33,6 @@ from models.policy import Allowlist, Policy from services.API import AirlockAPIWrapper from TUI.Widgets.policyselector import PolicySelector from utils.configmanager import get_system_list, get_system_value, load_env -from utils.utils import formatHTML logger = logging.getLogger(__name__) @@ -64,12 +63,20 @@ class PolicyPrepWorkflowScreen(Screen): DEFAULT_CSS = """ DataTable > .datatable--row.selected { - background: $primary 30%; + background: $accent; + color: $background; } - + + /* Highlighted cursor row */ DataTable:focus > .datatable--cursor { background: $secondary 20%; } + + /* When a row is both selected and has cursor, selection wins */ + DataTable:focus > .datatable--cursor.selected { + background: $accent; + color: $background; + } #workflow_title { text-style: bold; @@ -117,6 +124,7 @@ class PolicyPrepWorkflowScreen(Screen): Binding("q", "main_menu", "Main Menu"), Binding("f", "open_folder", "Open Folder"), Binding("d", "delete_rows", "Delete Selected"), + Binding("c", "copy_rows", "Copy Selected"), Binding("a", "select_all", "Select All"), Binding("n", "select_none", "Select None"), Binding("space", "toggle_selection", "Toggle Selection", show=False), @@ -216,20 +224,21 @@ class PolicyPrepWorkflowScreen(Screen): status_widget = self.query_one("#workflow_status", Static) stage_messages = { + "introduction": "Step 0: Workflow Introduction", "select_source": "Step 1: Select Source Policies", "select_destination": "Step 2: Select Destination Policy", "select_allowlist": "Step 3: Select Destination Allowlist", "fetch_data": "Step 4: Fetch Execution History", - "fetching": "Fetching and sorting execution data...", + "fetching": "Processing: Fetching and sorting execution data...", "first_review": "Step 5: First Manual Review", - "build_paths": "Step 6: Building Path Exclusions", - "second_review": "Step 7: Second Manual Review", - "test": "Step 8: Test - Preview Changes", - "liftoff": "Step 9: Liftoff - Apply Changes", - "complete": "Workflow Complete", + "building_paths": "Processing: Building path exclusions and publishers...", + "second_review": "Step 6: Second Manual Review", + "test": "Step 7: Test - Preview Changes", + "liftoff": "Step 8: Liftoff - Apply Changes", + "complete": "βœ… Workflow Complete!", } - status_widget.update(stage_messages.get(self.workflow_stage, "Unknown Stage")) + status_widget.update(stage_messages.get(self.workflow_stage, "Processing...")) def _update_checklist(self) -> None: """Update the preparation checklist display.""" @@ -395,7 +404,7 @@ class PolicyPrepWorkflowScreen(Screen): " πŸ“‹ Step 1: Select source policies (data collection)\n" " 🎯 Step 2: Select destination policy (where changes go)\n" " πŸ“ Step 3: Select destination allowlist\n" - " πŸ“Š Step 4: Fetch execution data (may take 1-2 minutes)\n" + " πŸ“Š Step 4: Fetch execution data (5-30 minutes, depending on policy size)\n" " βœ… Step 5: Review approved/needs review executions\n" " πŸ“ Step 6: Review path exclusions and publishers\n" " πŸ” Step 7: Preview changes before applying\n" @@ -405,7 +414,9 @@ class PolicyPrepWorkflowScreen(Screen): content.mount(steps) # Time estimate - estimate = Static("⏱️ Estimated Time: 15-30 minutes depending on data size") + estimate = Static( + "⏱️ Estimated Time: 1-3 hours for large policies (policies with 100k+ executions may take longer)" + ) estimate.styles.margin = (1, 2) estimate.styles.color = "cyan" content.mount(estimate) @@ -479,6 +490,13 @@ class PolicyPrepWorkflowScreen(Screen): content.mount(table) + # CRITICAL: Prevent default first-row selection + # Move focus away from the table so no row is highlighted initially + try: + content.focus() + except Exception as e: + logger.debug(f"Could not clear table focus: {e}") + # Control buttons control_container = Horizontal() control_container.styles.height = "auto" @@ -663,7 +681,7 @@ class PolicyPrepWorkflowScreen(Screen): # Show notification that fetch is starting self.app.notify( - "Starting data fetch - this may take several minutes for large policies", + "Starting data fetch - this may take 5-30 minutes for large policies with millions of executions", severity="information", timeout=5, ) @@ -768,18 +786,18 @@ class PolicyPrepWorkflowScreen(Screen): f"{len(needs_review)} needs review, {len(unknown)} unknown" ) - # Store the data - self.approved_df = ( + # Store the data - convert all unhashable objects to strings + self.approved_df = self._sanitize_dataframe( pd.DataFrame([r.__dict__ for r in approved]) if approved else pd.DataFrame() ) - self.unapproved_df = ( + self.unapproved_df = self._sanitize_dataframe( pd.DataFrame([r.__dict__ for r in unapproved]) if unapproved else pd.DataFrame() ) - self.needs_review_df = ( + self.needs_review_df = self._sanitize_dataframe( pd.DataFrame([r.__dict__ for r in needs_review]) if needs_review else pd.DataFrame() @@ -798,17 +816,50 @@ class PolicyPrepWorkflowScreen(Screen): self.app.notify(f"Failed to fetch data: {str(e)}", severity="error") self._show_fetch_data() + def _sanitize_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: + """ + Convert any unhashable objects (like custom Hash objects) to strings. + This prevents 'unhashable type' errors in pandas operations like drop_duplicates(). + + Args: + df: DataFrame that may contain unhashable objects + + Returns: + Sanitized DataFrame with all objects converted to hashable types + """ + if df.empty: + return df + + df = df.copy() + for col in df.columns: + if df[col].dtype == "object": + try: + # Check if column contains unhashable custom objects + sample = df[col].iloc[0] if len(df) > 0 else None + if sample is not None: + # Try to hash it - if it fails, convert to string + try: + hash(sample) + except TypeError: + # Unhashable type - convert entire column to string + df[col] = df[col].astype(str) + logger.debug( + f"Converted unhashable column '{col}' to strings" + ) + except Exception as e: + logger.debug(f"Error checking column '{col}': {e}") + + return df + def _save_fetched_data(self) -> None: - """Save fetched data to CSV and HTML files.""" + """Save fetched data to CSV files.""" if not self.source_policies: return policy_name = self.source_policies[0].name review_dir = os.path.join(self.working_dir, "Needs_Review", "Review_First") - html_dir = os.path.join(self.working_dir, "Needs_Review", "HTML") os.makedirs(review_dir, exist_ok=True) - os.makedirs(html_dir, exist_ok=True) # Save each category categories = { @@ -822,10 +873,8 @@ class PolicyPrepWorkflowScreen(Screen): csv_path = os.path.join( review_dir, f"{policy_name}_{label}_executions.csv" ) - html_path = os.path.join(html_dir, f"{policy_name}_{label}.html") df.to_csv(csv_path, index=False) - formatHTML(df, html_path) logger.info(f"Saved {label} executions to {csv_path}") @@ -899,10 +948,51 @@ class PolicyPrepWorkflowScreen(Screen): self._creating_review_table = True + # Disable all tab buttons to prevent race conditions + self._disable_tab_buttons() + try: self._show_review_table_impl(table_type) finally: self._creating_review_table = False + # Re-enable tab buttons after table is created + self._enable_tab_buttons() + + def _disable_tab_buttons(self) -> None: + """Disable all tab switching buttons to prevent race conditions.""" + try: + content = self.query_one("#content_area", Vertical) + tab_buttons = content.query("Button") + for btn in tab_buttons: + if btn.id in [ + "show_approved_tab", + "show_needs_review_tab", + "show_paths_tab", + "show_publishers_tab", + "show_remaining_tab", + ]: + btn.disabled = True + logger.debug(f"Disabled button: {btn.id}") + except Exception as e: + logger.debug(f"Error disabling tab buttons: {e}") + + def _enable_tab_buttons(self) -> None: + """Re-enable all tab switching buttons.""" + try: + content = self.query_one("#content_area", Vertical) + tab_buttons = content.query("Button") + for btn in tab_buttons: + if btn.id in [ + "show_approved_tab", + "show_needs_review_tab", + "show_paths_tab", + "show_publishers_tab", + "show_remaining_tab", + ]: + btn.disabled = False + logger.debug(f"Enabled button: {btn.id}") + except Exception as e: + logger.debug(f"Error enabling tab buttons: {e}") def _show_review_table_impl(self, table_type: str) -> None: """Internal implementation of _show_review_table.""" @@ -937,41 +1027,52 @@ class PolicyPrepWorkflowScreen(Screen): self.needs_review_df = df # Remove the SPECIFIC table we're about to create if it exists + # Buttons are now disabled during creation, so this should be quick try: existing_specific = content.query_one(f"#{table_id}", DataTable) if existing_specific: logger.debug(f"Removing existing table with ID: {table_id}") existing_specific.remove() + content.refresh(layout=True) except Exception as e: - logger.debug( - f"No existing table with ID {table_id} found (this is normal): {e}" - ) + # Table doesn't exist - good! + logger.debug(f"No existing table with ID {table_id} found: {e}") # Remove ALL existing DataTables to be safe try: existing_tables = content.query("DataTable") - logger.debug(f"Found {len(existing_tables)} existing tables to remove") - for table in existing_tables: - logger.debug(f"Removing table: {table.id}") - table.remove() + if existing_tables: + logger.debug(f"Found {len(existing_tables)} existing tables to remove") + for table in existing_tables: + logger.debug(f"Removing table: {table.id}") + try: + table.remove() + except Exception as e: + logger.debug(f"Error removing table {table.id}: {e}") + content.refresh(layout=True) except Exception as e: - logger.debug(f"Error removing existing tables: {e}") + logger.debug(f"Error querying/removing tables: {e}") - # Remove existing instruction and help text (they accumulate without removal) - # Remove ALL Static widgets - they're just text that needs to be replaced + # Remove existing instruction and help text try: existing_statics = content.query("Static") logger.debug( f"Found {len(existing_statics)} existing Static widgets to remove" ) for static in existing_statics: - static.remove() + try: + static.remove() + except Exception as e: + logger.debug(f"Error removing Static: {e}") except Exception as e: logger.debug(f"Error removing Static widgets: {e}") - # Force a refresh to ensure removals are processed + # Final refresh to ensure all removals are processed try: - content.refresh() + content.refresh(layout=True) + import time + + time.sleep(0.2) # Give DOM time to process removals (200ms) except Exception as e: logger.debug(f"Error refreshing content: {e}") @@ -998,29 +1099,75 @@ class PolicyPrepWorkflowScreen(Screen): # Help text (no ID needed) help_text = Static( "Click to toggle, 'r' for range select (click start, press 'r', click end)\n" - "Space to toggle cursor row, 'd' to delete, 'a' select all, arrows navigate" + "Space to toggle cursor row, 'd' to delete, 'c' to copy, 'a' select all, arrows navigate" ) help_text.styles.margin = (0, 1, 1, 1) help_text.styles.text_style = "dim" content.mount(help_text) - # CRITICAL: Check if table already exists in content (should not happen after removal above) + # Final safety check - WAIT until table is confirmed gone + # Use longer waits (1-2 seconds) to ensure DOM has time to process + max_wait_attempts = 5 + for wait_attempt in range(max_wait_attempts): + try: + existing_check = content.query_one(f"#{table_id}", DataTable) + if existing_check: + logger.warning( + f"Table {table_id} still exists (attempt {wait_attempt + 1}/{max_wait_attempts}). Removing and waiting..." + ) + existing_check.remove() + content.refresh(layout=True) + import time + + time.sleep(0.5) # Wait 0.5s for DOM to process + else: + # Table is gone, break out + logger.debug( + f"Table {table_id} confirmed removed after {wait_attempt} attempts" + ) + break + except Exception: + # Good - table doesn't exist, break out + logger.debug(f"Table {table_id} not found (good)") + break + + # Final verification - if table STILL exists after all attempts, force remove it and wait longer try: - existing_check = content.query_one(f"#{table_id}", DataTable) - if existing_check: + final_check = content.query_one(f"#{table_id}", DataTable) + if final_check: logger.error( - f"Table {table_id} STILL EXISTS after removal! This should not happen." + f"CRITICAL: Table {table_id} still exists after {max_wait_attempts} attempts!" ) - # Don't create a new one - just return - return + final_check.remove() + content.refresh(layout=True) + import time + + time.sleep(1.0) # Wait a full second + + # Check one more time + try: + still_there = content.query_one(f"#{table_id}", DataTable) + if still_there: + # This should never happen - log and skip mounting + logger.error( + f"FATAL: Cannot remove {table_id} even after 1 second wait. Skipping mount to prevent DuplicateIds." + ) + self.app.notify( + "Table refresh failed. Please try again.", severity="error" + ) + return + except Exception: + # Good - finally gone + logger.info(f"Table {table_id} finally removed after extended wait") + pass except Exception: - # Good - table doesn't exist, proceed with creation + # Good - table doesn't exist pass # Create the review table review_table = DataTable(id=table_id) review_table.styles.height = "40vh" # Increased since we removed button rows - review_table.cursor_type = "row" + review_table.cursor_type = "row" # Need row cursor for clicking/navigation review_table.zebra_stripes = True # Specified columns in order @@ -1055,10 +1202,13 @@ class PolicyPrepWorkflowScreen(Screen): final_check = content.query_one(f"#{table_id}", DataTable) if final_check: logger.warning( - f"Table {table_id} still exists after removal attempts! Forcing removal..." + f"Table {table_id} still exists. Waiting and removing..." ) final_check.remove() - content.refresh() + content.refresh(layout=True) + import time + + time.sleep(0.05) except Exception: # Good - table doesn't exist pass @@ -1066,6 +1216,17 @@ class PolicyPrepWorkflowScreen(Screen): content.mount(review_table) logger.debug(f"Successfully mounted {table_id} with {len(df)} rows") + # CRITICAL: Prevent default first-row selection + # Textual auto-focuses new DataTables, causing first row to be selected + # Solution: temporarily disable focus, then re-enable + try: + review_table.can_focus = False + # Schedule re-enabling focus after UI settles + self.set_timer(0.1, lambda: setattr(review_table, "can_focus", True)) + logger.debug("Disabled initial table focus to prevent default selection") + except Exception as e: + logger.debug(f"Could not prevent default focus: {e}") + # Row count display only (removed Select All, Clear, Delete buttons) try: control_container = content.query_one("#review_controls", Horizontal) @@ -1179,6 +1340,71 @@ class PolicyPrepWorkflowScreen(Screen): else: self.app.notify(f"Deleted {removed_count} rows", severity="information") + def _copy_selected_rows(self) -> None: + """Copy selected rows to clipboard as tab-separated values.""" + if not hasattr(self, "selected_rows") or not self.selected_rows: + self.app.notify("No rows selected to copy", severity="warning") + return + + # Determine which dataframe to use + if self.current_review_type == "approved": + df = self.approved_df + else: + df = self.needs_review_df + + if df is None: + return + + # Get selected rows + indices_to_copy = [int(idx) for idx in self.selected_rows] + selected_df = df.loc[df.index.isin(indices_to_copy)] + + if selected_df.empty: + self.app.notify("No valid rows to copy", severity="warning") + return + + # Convert to dictionary format (list of dicts) + try: + # Convert DataFrame to list of dictionaries + rows_as_dicts = selected_df.to_dict("records") + + # Format as Python dictionary representation + import json + + dict_data = json.dumps(rows_as_dicts, indent=2) + + # Copy to clipboard + import platform + import subprocess + + if platform.system() == "Windows": + # Windows clipboard + subprocess.run(["clip"], input=dict_data.encode("utf-8"), check=True) + elif platform.system() == "Darwin": + # macOS clipboard + subprocess.run(["pbcopy"], input=dict_data.encode("utf-8"), check=True) + else: + # Linux clipboard (try xclip first, then xsel) + try: + subprocess.run( + ["xclip", "-selection", "clipboard"], + input=dict_data.encode("utf-8"), + check=True, + ) + except FileNotFoundError: + subprocess.run( + ["xsel", "--clipboard", "--input"], + input=dict_data.encode("utf-8"), + check=True, + ) + + self.app.notify( + f"Copied {len(selected_df)} rows as JSON", severity="information" + ) + except Exception as e: + logger.error(f"Failed to copy to clipboard: {e}") + self.app.notify(f"Failed to copy: {str(e)}", severity="error") + def _show_path_building_screen(self) -> None: """Show loading screen before building paths and publishers.""" self.workflow_stage = "building_paths" @@ -1218,27 +1444,40 @@ class PolicyPrepWorkflowScreen(Screen): if not self.source_policies: raise ValueError("No source policies selected") - policy_name = self.source_policies[0].name - approved_path = os.path.join( - self.working_dir, "Approved", f"{policy_name}_approved_executions.csv" - ) - review_path = os.path.join( - self.working_dir, - "Approved", - f"{policy_name}_needs_review_executions.csv", + # CRITICAL FIX: Use current in-memory DataFrames, not stale CSV files + # This ensures that any deletions made in the review step are reflected + df1 = self.approved_df if self.approved_df is not None else pd.DataFrame() + df2 = ( + self.needs_review_df + if self.needs_review_df is not None + else pd.DataFrame() ) - # Load approved files - df1 = ( - pd.read_csv(approved_path) - if os.path.exists(approved_path) - else pd.DataFrame() - ) - df2 = ( - pd.read_csv(review_path) - if os.path.exists(review_path) - else pd.DataFrame() - ) + # If DataFrames are empty, try loading from saved files as fallback + if df1.empty and df2.empty: + logger.info("DataFrames empty, attempting to load from saved files...") + policy_name = self.source_policies[0].name + approved_path = os.path.join( + self.working_dir, + "Approved", + f"{policy_name}_approved_executions.csv", + ) + review_path = os.path.join( + self.working_dir, + "Approved", + f"{policy_name}_needs_review_executions.csv", + ) + + df1 = ( + pd.read_csv(approved_path) + if os.path.exists(approved_path) + else pd.DataFrame() + ) + df2 = ( + pd.read_csv(review_path) + if os.path.exists(review_path) + else pd.DataFrame() + ) if df1.empty and df2.empty: self.app.notify( @@ -1314,6 +1553,24 @@ class PolicyPrepWorkflowScreen(Screen): self.publishers_df = publist + # Sort publishers alphabetically + if ( + not self.publishers_df.empty + and "publisher" in self.publishers_df.columns + ): + self.publishers_df = self.publishers_df.sort_values( + by="publisher", ascending=True + ).reset_index(drop=True) + + # Sort publishers alphabetically + if ( + not self.publishers_df.empty + and "publisher" in self.publishers_df.columns + ): + self.publishers_df = self.publishers_df.sort_values( + by="publisher", ascending=True + ).reset_index(drop=True) + # Save to Review_Second folder self._save_path_data() @@ -1435,7 +1692,7 @@ class PolicyPrepWorkflowScreen(Screen): result = pd.DataFrame(new_rows).drop(columns=["group_key"]) logger.info( - f"_split_filepaths_grouped: Processed {len(df)} rows β†’ {len(result)} rows with metadata" + f"_split_filepaths_grouped: Processed {len(df)} rows Ò†’ {len(result)} rows with metadata" ) return result @@ -1563,10 +1820,8 @@ class PolicyPrepWorkflowScreen(Screen): policy_name = self.source_policies[0].name review_dir = os.path.join(self.working_dir, "Needs_Review", "Review_Second") - html_dir = os.path.join(self.working_dir, "Needs_Review", "HTML") os.makedirs(review_dir, exist_ok=True) - os.makedirs(html_dir, exist_ok=True) # Save each dataframe dataframes = { @@ -1579,13 +1834,27 @@ class PolicyPrepWorkflowScreen(Screen): for name, df in dataframes.items(): if df is not None and not df.empty: csv_path = os.path.join(review_dir, f"{policy_name}_{name}.csv") - html_path = os.path.join(html_dir, f"{policy_name}_{name}.html") df.to_csv(csv_path, index=False) - formatHTML(df, html_path) logger.info(f"Saved {name} to {csv_path}") + def _show_fetch_data_after_clearing_paths(self) -> None: + """Navigate back to fetch data screen and clear path data for regeneration.""" + logger.info( + "Going back to first review - clearing path data to force regeneration" + ) + # Clear path-related dataframes so they get regenerated with current data + self.primary_paths_df = None + self.secondary_paths_df = None + self.publishers_df = None + self.remaining_hashes_df = None + # Reset path review tracking + self.paths_tab_reviewed = False + self.publishers_tab_reviewed = False + # Now show the fetch data screen + self._show_fetch_data() + def _show_path_results(self) -> None: """Show the results of path building.""" logger.debug("=== _show_path_results called ===") @@ -1652,10 +1921,15 @@ class PolicyPrepWorkflowScreen(Screen): self._creating_path_table = True + # Disable all tab buttons to prevent race conditions + self._disable_tab_buttons() + try: self._show_path_review_table_impl(table_type) finally: self._creating_path_table = False + # Re-enable tab buttons after table is created + self._enable_tab_buttons() def _show_path_review_table_impl(self, table_type: str) -> None: """Internal implementation of _show_path_review_table.""" @@ -1813,7 +2087,7 @@ class PolicyPrepWorkflowScreen(Screen): if table_type != "remaining": help_text = Static( "Click to toggle, 'r' for range select (click start, press 'r', click end)\n" - "Space to toggle cursor row, 'd' to delete, 'a' select all, arrows navigate" + "Space to toggle cursor row, 'd' to delete, 'c' to copy, 'a' select all, arrows navigate" ) else: help_text = Static( @@ -1827,7 +2101,7 @@ class PolicyPrepWorkflowScreen(Screen): # Create the review table review_table = DataTable(id=table_id) review_table.styles.height = "35vh" # Increased since we removed button rows - review_table.cursor_type = "row" + review_table.cursor_type = "row" # Need row cursor for clicking/navigation review_table.zebra_stripes = True # Add columns - checkbox first, then data columns @@ -1854,19 +2128,79 @@ class PolicyPrepWorkflowScreen(Screen): logger.debug(f"About to mount {table_id}") - # Final safety check before mounting table + # Final safety check - WAIT until table is confirmed gone before mounting + # Use longer waits (1-2 seconds) to ensure DOM has time to process + max_wait_attempts = 5 + for wait_attempt in range(max_wait_attempts): + try: + existing_check = content.query_one(f"#{table_id}", DataTable) + if existing_check: + logger.warning( + f"Table {table_id} still exists (attempt {wait_attempt + 1}/{max_wait_attempts}). Removing and waiting..." + ) + existing_check.remove() + content.refresh(layout=True) + import time + + time.sleep(0.5) # Wait 0.5s for DOM to process + else: + # Table is gone, break out + logger.debug( + f"Table {table_id} confirmed removed after {wait_attempt} attempts" + ) + break + except Exception: + # Good - table doesn't exist, break out + logger.debug(f"Table {table_id} not found (good)") + break + + # Final verification - if table STILL exists after all attempts, force remove it and wait longer try: final_check = content.query_one(f"#{table_id}", DataTable) if final_check: - logger.warning(f"Table {table_id} still exists! Forcing removal...") + logger.error( + f"CRITICAL: Table {table_id} still exists after {max_wait_attempts} attempts!" + ) final_check.remove() - content.refresh() + content.refresh(layout=True) + import time + + time.sleep(1.0) # Wait a full second + + # Check one more time + try: + still_there = content.query_one(f"#{table_id}", DataTable) + if still_there: + # This should never happen - log and skip mounting + logger.error( + f"FATAL: Cannot remove {table_id} even after 1 second wait. Skipping mount to prevent DuplicateIds." + ) + self.app.notify( + "Table refresh failed. Please try again.", severity="error" + ) + return + except Exception: + # Good - finally gone + logger.info(f"Table {table_id} finally removed after extended wait") + pass except Exception: + # Good - table doesn't exist pass content.mount(review_table) logger.debug(f"Successfully mounted {table_id}") + # CRITICAL: Prevent default first-row selection + # Textual auto-focuses new DataTables, causing first row to be selected + # Solution: temporarily disable focus, then re-enable + try: + review_table.can_focus = False + # Schedule re-enabling focus after UI settles + self.set_timer(0.1, lambda: setattr(review_table, "can_focus", True)) + logger.debug("Disabled initial table focus to prevent default selection") + except Exception as e: + logger.debug(f"Could not prevent default focus: {e}") + # Row count display only (removed Select All, Clear, Delete buttons) if table_type != "remaining": # Check if controls container already exists, reuse if it does @@ -2012,6 +2346,90 @@ class PolicyPrepWorkflowScreen(Screen): f"Deleted {len(indices_to_delete)} items", severity="information" ) + def _copy_selected_path_rows(self) -> None: + """Copy selected path/publisher rows to clipboard as tab-separated values.""" + if not hasattr(self, "selected_path_rows") or not self.selected_path_rows: + self.app.notify("No rows selected to copy", severity="warning") + return + + # Determine which dataframe to use + df = None + if self.current_path_review_type == "paths": + # Combine primary and secondary paths + dfs = [] + if self.primary_paths_df is not None and not self.primary_paths_df.empty: + df_copy = self.primary_paths_df.copy() + df_copy["type"] = "primary" + dfs.append(df_copy) + if ( + self.secondary_paths_df is not None + and not self.secondary_paths_df.empty + ): + df_copy = self.secondary_paths_df.copy() + df_copy["type"] = "secondary" + dfs.append(df_copy) + + if dfs: + df = pd.concat(dfs, ignore_index=True) + elif self.current_path_review_type == "publishers": + df = self.publishers_df + else: + df = self.remaining_hashes_df + + if df is None or df.empty: + self.app.notify("No data to copy", severity="warning") + return + + # Get selected rows + indices_to_copy = [int(idx) for idx in self.selected_path_rows] + selected_df = df.loc[df.index.isin(indices_to_copy)] + + if selected_df.empty: + self.app.notify("No valid rows to copy", severity="warning") + return + + # Convert to dictionary format (list of dicts) + try: + # Convert DataFrame to list of dictionaries + rows_as_dicts = selected_df.to_dict("records") + + # Format as Python dictionary representation + import json + + dict_data = json.dumps(rows_as_dicts, indent=2) + + # Copy to clipboard + import platform + import subprocess + + if platform.system() == "Windows": + # Windows clipboard + subprocess.run(["clip"], input=dict_data.encode("utf-8"), check=True) + elif platform.system() == "Darwin": + # macOS clipboard + subprocess.run(["pbcopy"], input=dict_data.encode("utf-8"), check=True) + else: + # Linux clipboard (try xclip first, then xsel) + try: + subprocess.run( + ["xclip", "-selection", "clipboard"], + input=dict_data.encode("utf-8"), + check=True, + ) + except FileNotFoundError: + subprocess.run( + ["xsel", "--clipboard", "--input"], + input=dict_data.encode("utf-8"), + check=True, + ) + + self.app.notify( + f"Copied {len(selected_df)} rows as JSON", severity="information" + ) + except Exception as e: + logger.error(f"Failed to copy to clipboard: {e}") + self.app.notify(f"Failed to copy: {str(e)}", severity="error") + def _build_preflight(self) -> None: """Build preflight files for testing.""" try: @@ -2028,10 +2446,20 @@ class PolicyPrepWorkflowScreen(Screen): content = self.query_one("#content_area", Vertical) content.remove_children() - summary = Static( - "Test Mode - Preview Changes\n\n" - "Review the paths that will be added to your policy:" - ) + # Check what we're actually applying + has_paths = ( + self.primary_paths_df is not None and not self.primary_paths_df.empty + ) or (self.secondary_paths_df is not None and not self.secondary_paths_df.empty) + has_publishers = self.publishers_df is not None and not self.publishers_df.empty + has_hashes = self.approved_df is not None and not self.approved_df.empty + + # Dynamic summary based on what we have + if has_paths or has_publishers: + summary_text = "Test Mode - Preview Changes\n\nReview the paths that will be added to your policy:" + else: + summary_text = "Test Mode - Preview Changes\n\nReview the hash approvals that will be added to your allowlist:" + + summary = Static(summary_text) summary.styles.margin = (1, 1) summary.styles.text_style = "bold" content.mount(summary) @@ -2083,10 +2511,22 @@ class PolicyPrepWorkflowScreen(Screen): # Show hash count if self.approved_df is not None and not self.approved_df.empty: - hash_info = Static( - f"\nπŸ” Individual Hash Approvals: {len(self.approved_df):,} hashes\n" - f" (Files not covered by paths or publishers)" + # Check if hashes are the only thing being applied + has_other_rules = ( + (self.primary_paths_df is not None and not self.primary_paths_df.empty) + or ( + self.secondary_paths_df is not None + and not self.secondary_paths_df.empty + ) + or (self.publishers_df is not None and not self.publishers_df.empty) ) + + if has_other_rules: + hash_text = f"\nπŸ” Individual Hash Approvals: {len(self.approved_df):,} hashes\n (Files not covered by paths or publishers)" + else: + hash_text = f"\nπŸ” Individual Hash Approvals: {len(self.approved_df):,} hashes\n (All approved files will be added by hash)" + + hash_info = Static(hash_text) hash_info.styles.margin = (1, 1) content.mount(hash_info) @@ -2106,7 +2546,7 @@ class PolicyPrepWorkflowScreen(Screen): content.mount(button_container) back_btn = Button( - "← Back to Review", id="back_to_path_review", variant="default" + "Ò† Back to Review", id="back_to_path_review", variant="default" ) liftoff_btn = Button( "Liftoff - Apply Changes πŸš€", id="liftoff", variant="success" @@ -2365,7 +2805,7 @@ class PolicyPrepWorkflowScreen(Screen): try: table = self.query_one(f"#{table_id}", DataTable) - # Update checkboxes in place without rebuilding the table + # Update checkboxes and apply styling to all cells in selected rows row_index = 0 for row_key in table.rows.keys(): # Get the actual value from the RowKey object @@ -2382,8 +2822,55 @@ class PolicyPrepWorkflowScreen(Screen): except Exception as e: logger.error(f"Could not update cell at row {row_index}: {e}") + # CRITICAL: Apply visual styling to ALL cells in the row if selected + # This creates the visual "highlight" effect for multi-select + try: + if is_selected: + # Get the number of columns + num_cols = len(table.columns) + # Update each cell with Rich styling for background color + for col_idx in range(num_cols): + try: + # Get current cell value + current_value = str( + table.get_cell_at((row_index, col_idx)) + ) + # Wrap in Rich markup for background color + # Using reverse video to invert colors + styled_value = f"[reverse]{current_value}[/reverse]" + table.update_cell_at((row_index, col_idx), styled_value) + except Exception as cell_err: + logger.debug( + f"Could not style cell ({row_index}, {col_idx}): {cell_err}" + ) + else: + # Remove styling from deselected rows + num_cols = len(table.columns) + for col_idx in range(num_cols): + try: + current_value = str( + table.get_cell_at((row_index, col_idx)) + ) + # Remove Rich markup if present + if current_value.startswith("[reverse]"): + clean_value = current_value.replace( + "[reverse]", "" + ).replace("[/reverse]", "") + table.update_cell_at( + (row_index, col_idx), clean_value + ) + except Exception as cell_err: + logger.debug( + f"Could not unstyle cell ({row_index}, {col_idx}): {cell_err}" + ) + except Exception as style_err: + logger.debug(f"Error styling row {row_index}: {style_err}") + row_index += 1 + # Refresh table to show changes + table.refresh() + except Exception as e: logger.error(f"Error refreshing table {table_id}: {e}", exc_info=True) @@ -2655,21 +3142,66 @@ class PolicyPrepWorkflowScreen(Screen): else: self._show_destination_policy_selection() - # Tab switching buttons + # Tab switching buttons - disable immediately to prevent double-clicks elif button_id == "show_approved_tab": - self._show_review_table("approved") + # Prevent rapid clicking - check if table creation is already in progress + if self._creating_review_table: + logger.debug("Ignoring tab click - table creation already in progress") + return + # Disable this button immediately + event.button.disabled = True + try: + self._show_review_table("approved") + finally: + event.button.disabled = False elif button_id == "show_needs_review_tab": - self._show_review_table("needs_review") + # Prevent rapid clicking - check if table creation is already in progress + if self._creating_review_table: + logger.debug("Ignoring tab click - table creation already in progress") + return + # Disable this button immediately + event.button.disabled = True + try: + self._show_review_table("needs_review") + finally: + event.button.disabled = False elif button_id == "show_paths_tab": - self._show_path_review_table("paths") + # Prevent rapid clicking - check if table creation is already in progress + if self._creating_path_table: + logger.debug("Ignoring tab click - table creation already in progress") + return + # Disable this button immediately + event.button.disabled = True + try: + self._show_path_review_table("paths") + finally: + event.button.disabled = False elif button_id == "show_publishers_tab": - self._show_path_review_table("publishers") + # Prevent rapid clicking - check if table creation is already in progress + if self._creating_path_table: + logger.debug("Ignoring tab click - table creation already in progress") + return + # Disable this button immediately + event.button.disabled = True + try: + self._show_path_review_table("publishers") + finally: + event.button.disabled = False elif button_id == "show_remaining_tab": - self._show_path_review_table("remaining") + # Prevent rapid clicking - check if table creation is already in progress + if self._creating_path_table: + logger.debug("Ignoring tab click - table creation already in progress") + return + # Disable this button immediately + event.button.disabled = True + try: + self._show_path_review_table("remaining") + finally: + event.button.disabled = False # Row selection buttons elif button_id == "select_all_rows": @@ -2756,6 +3288,9 @@ class PolicyPrepWorkflowScreen(Screen): ) else: # Save the reviewed data before continuing + logger.info( + f"Saving reviewed data: approved={len(self.approved_df) if self.approved_df is not None else 0}, needs_review={len(self.needs_review_df) if self.needs_review_df is not None else 0}" + ) self._save_reviewed_data() # Show loading screen then build paths self._show_path_building_screen() @@ -2770,14 +3305,31 @@ class PolicyPrepWorkflowScreen(Screen): ) return - # Validate that path review is complete - if (self.primary_paths_df is None or self.primary_paths_df.empty) and ( - self.publishers_df is None or self.publishers_df.empty - ): + # Validate that we have SOMETHING to build with (paths, publishers, or hashes) + has_paths = ( + self.primary_paths_df is not None and not self.primary_paths_df.empty + ) or ( + self.secondary_paths_df is not None + and not self.secondary_paths_df.empty + ) + has_publishers = ( + self.publishers_df is not None and not self.publishers_df.empty + ) + has_hashes = self.approved_df is not None and not self.approved_df.empty + + if not has_paths and not has_publishers and not has_hashes: self.app.notify( - "No paths or publishers to build preflight with!", severity="error" + "No paths, publishers, or hashes to build preflight with!", + severity="error", ) else: + # Build with whatever we have + if not has_paths and not has_publishers and has_hashes: + self.app.notify( + f"Building preflight with {len(self.approved_df)} hash approvals only.", + severity="information", + timeout=3, + ) self._build_preflight() elif button_id == "back_to_path_review": @@ -3018,7 +3570,7 @@ class PolicyPrepWorkflowScreen(Screen): "select_destination": self._show_source_policy_selection, "select_allowlist": self._show_destination_policy_selection, "fetch_data": self._show_allowlist_selection, - "first_review": self._show_fetch_data, + "first_review": lambda: self._show_fetch_data_after_clearing_paths(), "second_review": self._show_fetch_results, "test": self._show_path_results, "complete": lambda: self.app.pop_screen(), @@ -3046,6 +3598,13 @@ class PolicyPrepWorkflowScreen(Screen): elif self.workflow_stage == "second_review": self._delete_selected_path_rows() + def action_copy_rows(self) -> None: + """Copy selected rows to clipboard.""" + if self.workflow_stage == "first_review": + self._copy_selected_rows() + elif self.workflow_stage == "second_review": + self._copy_selected_path_rows() + def action_select_all(self) -> None: """Select all rows in the current table.""" if self.workflow_stage == "first_review": diff --git a/TUI/Widgets/multiagentselector.py b/TUI/Widgets/multiagentselector.py index bc77706..b3603f3 100644 --- a/TUI/Widgets/multiagentselector.py +++ b/TUI/Widgets/multiagentselector.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import difflib +from pathlib import Path import re from typing import List, Optional @@ -57,7 +58,7 @@ class MultiAgentSelector(Widget): def compose(self): yield Header(show_clock=True, icon="βš™") - title_text = Static("πŸ–₯️ Agent Selector", id="selector_title") + title_text = Static("πŸ–₯️ Agent Selector", id="selector_title") title_text.styles.margin = (0, 0, 0, 1) yield title_text @@ -77,7 +78,7 @@ class MultiAgentSelector(Widget): text_area.styles.overflow_y = "auto" yield text_area - with Horizontal(id="switch_search_container"): + with Horizontal(id="switch_container"): switch = Switch(value=False, id="match_switch") switch.styles.width = "auto" switch.styles.margin = (1, 0, 0, 0) @@ -89,8 +90,13 @@ class MultiAgentSelector(Widget): switch_label.styles.margin = (2, 1, 0, 0) yield switch_label + with Horizontal(id="action_buttons_container"): + load_file = Button("πŸ“‚ Load File", id="load_file_button") + load_file.styles.margin = (1, 1, 0, 1) + yield load_file + search = Button("πŸ” Search", id="search_button") - search.styles.margin = (1, 0, 0, 0) + search.styles.margin = (1, 0, 0, 1) yield search with Horizontal() as select_buttons: @@ -152,6 +158,9 @@ class MultiAgentSelector(Widget): ] self.post_message(self.AgentsSelected(selected_agents)) event.stop() + elif btn_id == "load_file_button": + self._load_from_file() + event.stop() elif btn_id == "search_button": self.update_matches() event.stop() @@ -166,7 +175,7 @@ class MultiAgentSelector(Widget): match_list.add_option((name, name)) unmatched_label = self.query_one("#unmatched_label", Static) if unmatched: - unmatched_label.update(f"⚠️ No matches for: {', '.join(unmatched)}") + unmatched_label.update(f"Òő ï¸ No matches for: {', '.join(unmatched)}") else: unmatched_label.update("") @@ -214,3 +223,118 @@ class MultiAgentSelector(Widget): else: unmatched.append(name) return sorted(matched), unmatched + + def _load_from_file(self): + """Safely load device names from a text file.""" + try: + # Import here to avoid issues if tkinter isn't available + import tkinter as tk + from tkinter import filedialog + + # Create file dialog + root = tk.Tk() + root.withdraw() + + file_path = filedialog.askopenfilename( + title="Select device list file", + filetypes=[ + ("Text files", "*.txt"), + ("CSV files", "*.csv"), + ("All files", "*.*"), + ], + ) + + if not file_path: + # User cancelled + return + + # Validate file path + path_obj = Path(file_path) + if not path_obj.exists(): + self.app.notify("File does not exist", severity="error", timeout=3) + return + + if not path_obj.is_file(): + self.app.notify( + "Selected path is not a file", severity="error", timeout=3 + ) + return + + # Check file size (limit to 1 MB for safety) + file_size = path_obj.stat().st_size + if file_size > 1_000_000: # 1 MB + self.app.notify( + f"File too large ({file_size:,} bytes). Maximum 1 MB.", + severity="error", + timeout=5, + ) + return + + # Read file with proper encoding to preserve emojis + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + except UnicodeDecodeError: + # Try with different encoding if UTF-8 fails + try: + with open(file_path, "r", encoding="latin-1") as f: + content = f.read() + self.app.notify( + "File loaded with Latin-1 encoding (UTF-8 failed)", + severity="warning", + timeout=3, + ) + except Exception as e: + self.app.notify( + f"Error reading file: {str(e)}", severity="error", timeout=5 + ) + return + + # Validate and sanitize content + lines = content.split("\n") + valid_lines = [] + invalid_count = 0 + + # Pattern for valid hostnames/device names + # Allows: letters, numbers, hyphens, underscores, periods, and Unicode chars + hostname_pattern = re.compile(r"^[\w\-\.\u0080-\uFFFF]+$", re.UNICODE) + + for line in lines: + line = line.strip() + if not line: + continue # Skip empty lines + + # Check if line looks like a valid hostname/device name + if hostname_pattern.match(line): + valid_lines.append(line) + else: + invalid_count += 1 + # Log but don't add invalid entries + + if not valid_lines: + self.app.notify( + "No valid device names found in file", severity="warning", timeout=3 + ) + return + + # Update text area with validated content + text_area = self.query_one("#device_input", TextArea) + text_area.text = "\n".join(valid_lines) + + # Show notification + msg = f"βœ… Loaded {len(valid_lines)} devices from file" + if invalid_count > 0: + msg += f" ({invalid_count} invalid entries skipped)" + + self.app.notify(msg, severity="information", timeout=5) + + except ImportError: + self.app.notify( + "tkinter not available - cannot open file dialog", + severity="error", + timeout=3, + ) + except Exception as e: + self.app.notify( + f"Error loading file: {str(e)}", severity="error", timeout=5 + ) diff --git a/TUI/Widgets/serverlogwidget.py b/TUI/Widgets/serverlogwidget.py index 79ef132..0402700 100644 --- a/TUI/Widgets/serverlogwidget.py +++ b/TUI/Widgets/serverlogwidget.py @@ -19,7 +19,7 @@ import logging from bson import ObjectId from textual.app import ComposeResult from textual.containers import Container, Vertical -from textual.widgets import Button, DataTable, Static +from textual.widgets import Button, DataTable, Input, Static from services.API import AirlockAPIWrapper @@ -67,6 +67,19 @@ class ServerLogWidget(Vertical): height: auto; layout: horizontal; padding: 1; + align: left middle; + } + + ServerLogWidget .filter_label { + width: auto; + height: 3; + content-align: left middle; + padding-right: 1; + } + + ServerLogWidget #filter_input { + width: 40; + margin-right: 1; } ServerLogWidget Button { @@ -77,11 +90,15 @@ class ServerLogWidget(Vertical): def __init__(self, api: AirlockAPIWrapper): super().__init__() self.api = api + self.all_logs = [] # Store all logs for filtering + self.columns = [] # Store column names def compose(self) -> ComposeResult: yield Static("Loading server logs (last 72 hours)...", id="status_bar") yield DataTable(id="server_log_table") with Container(id="button_container"): + yield Static("Filter:", classes="filter_label") + yield Input(placeholder="Filter (use * and ? wildcards)", id="filter_input") yield Button("Refresh", id="refresh_button", variant="primary") def on_mount(self) -> None: @@ -105,23 +122,28 @@ class ServerLogWidget(Vertical): if not logs: status.update("β„ΉΓ―ΒΈ No server logs found in the last 72 hours.") table.clear(columns=True) + self.all_logs = [] + self.columns = [] return + # Store all logs for filtering + self.all_logs = logs + # Clear existing data table.clear(columns=True) # Add columns based on the first log entry if logs: first_log = logs[0] - columns = [col for col in first_log.keys() if col != "checkpoint"] + self.columns = [col for col in first_log.keys() if col != "checkpoint"] - for col in columns: + for col in self.columns: table.add_column(col, key=col) # Add rows in reverse order so newest entries are at the top for log_entry in reversed(logs): row_data = [] - for col in columns: + for col in self.columns: value = log_entry.get(col, "") # Format datetime column to be more readable if col == "datetime" and value: @@ -143,12 +165,86 @@ class ServerLogWidget(Vertical): logger.info(f"Loaded {len(logs)} server log entries") else: status.update("β„ΉΓ―ΒΈ No log entries found.") + self.all_logs = [] + self.columns = [] except Exception as exc: error_msg = f"❌ Error loading server logs: {exc}" status.update(error_msg) logger.error(f"Failed to load server logs: {exc}", exc_info=True) table.clear(columns=True) + self.all_logs = [] + self.columns = [] + + def filter_logs(self, filter_text: str) -> None: + """Filter the logs based on the filter text with wildcard support.""" + import fnmatch + + table = self.query_one("#server_log_table", DataTable) + status = self.query_one("#status_bar", Static) + + if not self.all_logs: + return + + # Clear existing data + table.clear(columns=True) + + # Re-add columns + for col in self.columns: + table.add_column(col, key=col) + + # Filter logs + filtered_logs = [] + if filter_text.strip(): + filter_pattern = filter_text.strip().lower() + for log_entry in self.all_logs: + # Check if any field matches the filter pattern + match = False + for col in self.columns: + value = str(log_entry.get(col, "")).lower() + if fnmatch.fnmatch(value, filter_pattern): + match = True + break + if match: + filtered_logs.append(log_entry) + else: + # No filter, show all logs + filtered_logs = self.all_logs + + # Add filtered rows in reverse order + for log_entry in reversed(filtered_logs): + row_data = [] + for col in self.columns: + value = log_entry.get(col, "") + # Format datetime column to be more readable + if col == "datetime" and value: + try: + dt = datetime.datetime.fromisoformat( + str(value).replace("Z", "+00:00") + ) + value = dt.strftime("%Y-%m-%d %H:%M:%S") + except Exception: + pass + row_data.append(str(value)) + table.add_row(*row_data) + + if filter_text.strip(): + status.update( + f"βœ… Showing {len(filtered_logs)} of {len(self.all_logs)} log entries (filtered)" + ) + else: + status.update( + f"βœ… Loaded {len(self.all_logs)} log entries from the last 72 hours" + ) + + logger.info( + f"Filtered to {len(filtered_logs)} entries with pattern: {filter_text}" + ) + + def on_input_changed(self, event: Input.Changed) -> None: + """Handle filter input changes.""" + if event.input.id == "filter_input": + self.filter_logs(event.value) def on_button_pressed(self, event: Button.Pressed) -> None: """Handle button presses.""" @@ -156,4 +252,10 @@ class ServerLogWidget(Vertical): if button_id == "refresh_button": self.load_logs() + # Clear the filter input when refreshing + try: + filter_input = self.query_one("#filter_input", Input) + filter_input.value = "" + except Exception: + pass event.stop()